2024-11-22 18:55:18,850 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-22 18:55:18,863 main DEBUG Took 0.011436 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-22 18:55:18,864 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-22 18:55:18,864 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-22 18:55:18,865 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-22 18:55:18,866 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 18:55:18,874 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-22 18:55:18,888 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 18:55:18,890 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 18:55:18,891 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 18:55:18,891 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 18:55:18,892 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 18:55:18,892 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 18:55:18,893 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 18:55:18,894 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 18:55:18,894 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 18:55:18,895 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 18:55:18,896 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 18:55:18,896 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 18:55:18,897 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 18:55:18,897 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 18:55:18,898 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 18:55:18,898 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 18:55:18,899 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 18:55:18,899 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 18:55:18,900 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 18:55:18,901 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 18:55:18,901 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 18:55:18,902 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 18:55:18,902 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 18:55:18,903 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 18:55:18,903 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 18:55:18,904 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-22 18:55:18,905 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 18:55:18,907 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-22 18:55:18,909 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-22 18:55:18,910 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-22 18:55:18,911 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-22 18:55:18,912 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-22 18:55:18,924 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-22 18:55:18,927 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-22 18:55:18,929 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-22 18:55:18,930 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-22 18:55:18,930 main DEBUG createAppenders(={Console}) 2024-11-22 18:55:18,931 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-22 18:55:18,932 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-22 18:55:18,932 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-22 18:55:18,933 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-22 18:55:18,933 main DEBUG OutputStream closed 2024-11-22 18:55:18,933 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-22 18:55:18,934 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-22 18:55:18,934 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-22 18:55:19,024 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-22 18:55:19,027 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-22 18:55:19,029 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-22 18:55:19,030 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-22 18:55:19,031 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-22 18:55:19,031 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-22 18:55:19,032 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-22 18:55:19,032 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-22 18:55:19,033 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-22 18:55:19,033 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-22 18:55:19,034 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-22 18:55:19,034 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-22 18:55:19,035 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-22 18:55:19,035 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-22 18:55:19,036 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-22 18:55:19,036 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-22 18:55:19,036 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-22 18:55:19,037 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-22 18:55:19,041 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-22 18:55:19,041 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-22 18:55:19,041 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-22 18:55:19,042 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-22T18:55:19,322 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/001b4aa4-1a7f-6591-3cef-5770bc0658f1 2024-11-22 18:55:19,326 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-22 18:55:19,326 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-22T18:55:19,335 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-11-22T18:55:19,359 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-22T18:55:19,375 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/001b4aa4-1a7f-6591-3cef-5770bc0658f1/cluster_d9590100-86d5-54e6-c888-90717c40c23c, deleteOnExit=true 2024-11-22T18:55:19,375 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-22T18:55:19,376 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/001b4aa4-1a7f-6591-3cef-5770bc0658f1/test.cache.data in system properties and HBase conf 2024-11-22T18:55:19,377 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/001b4aa4-1a7f-6591-3cef-5770bc0658f1/hadoop.tmp.dir in system properties and HBase conf 2024-11-22T18:55:19,377 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/001b4aa4-1a7f-6591-3cef-5770bc0658f1/hadoop.log.dir in system properties and HBase conf 2024-11-22T18:55:19,378 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/001b4aa4-1a7f-6591-3cef-5770bc0658f1/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-22T18:55:19,379 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/001b4aa4-1a7f-6591-3cef-5770bc0658f1/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-22T18:55:19,379 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-22T18:55:19,478 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-22T18:55:19,577 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T18:55:19,582 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/001b4aa4-1a7f-6591-3cef-5770bc0658f1/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-22T18:55:19,583 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/001b4aa4-1a7f-6591-3cef-5770bc0658f1/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-22T18:55:19,584 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/001b4aa4-1a7f-6591-3cef-5770bc0658f1/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-22T18:55:19,584 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/001b4aa4-1a7f-6591-3cef-5770bc0658f1/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-22T18:55:19,585 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/001b4aa4-1a7f-6591-3cef-5770bc0658f1/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-22T18:55:19,585 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/001b4aa4-1a7f-6591-3cef-5770bc0658f1/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-22T18:55:19,586 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/001b4aa4-1a7f-6591-3cef-5770bc0658f1/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-22T18:55:19,586 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/001b4aa4-1a7f-6591-3cef-5770bc0658f1/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-22T18:55:19,587 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/001b4aa4-1a7f-6591-3cef-5770bc0658f1/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-22T18:55:19,588 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/001b4aa4-1a7f-6591-3cef-5770bc0658f1/nfs.dump.dir in system properties and HBase conf 2024-11-22T18:55:19,588 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/001b4aa4-1a7f-6591-3cef-5770bc0658f1/java.io.tmpdir in system properties and HBase conf 2024-11-22T18:55:19,589 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/001b4aa4-1a7f-6591-3cef-5770bc0658f1/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-22T18:55:19,589 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/001b4aa4-1a7f-6591-3cef-5770bc0658f1/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-22T18:55:19,590 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/001b4aa4-1a7f-6591-3cef-5770bc0658f1/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-22T18:55:20,508 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-22T18:55:20,598 INFO [Time-limited test {}] log.Log(170): Logging initialized @2570ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-22T18:55:20,685 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-22T18:55:20,758 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-22T18:55:20,787 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-22T18:55:20,787 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-22T18:55:20,789 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-22T18:55:20,806 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-22T18:55:20,809 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/001b4aa4-1a7f-6591-3cef-5770bc0658f1/hadoop.log.dir/,AVAILABLE} 2024-11-22T18:55:20,810 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-22T18:55:21,064 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/001b4aa4-1a7f-6591-3cef-5770bc0658f1/java.io.tmpdir/jetty-localhost-35835-hadoop-hdfs-3_4_1-tests_jar-_-any-10847417592510163910/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-22T18:55:21,072 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:35835} 2024-11-22T18:55:21,072 INFO [Time-limited test {}] server.Server(415): Started @3046ms 2024-11-22T18:55:21,491 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-22T18:55:21,500 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-22T18:55:21,501 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-22T18:55:21,501 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-22T18:55:21,502 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-22T18:55:21,503 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/001b4aa4-1a7f-6591-3cef-5770bc0658f1/hadoop.log.dir/,AVAILABLE} 2024-11-22T18:55:21,503 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-22T18:55:21,637 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/001b4aa4-1a7f-6591-3cef-5770bc0658f1/java.io.tmpdir/jetty-localhost-39679-hadoop-hdfs-3_4_1-tests_jar-_-any-6573718651550351735/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-22T18:55:21,638 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:39679} 2024-11-22T18:55:21,638 INFO [Time-limited test {}] server.Server(415): Started @3612ms 2024-11-22T18:55:21,702 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-22T18:55:22,232 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/001b4aa4-1a7f-6591-3cef-5770bc0658f1/cluster_d9590100-86d5-54e6-c888-90717c40c23c/data/data2/current/BP-1295690749-172.17.0.2-1732301720241/current, will proceed with Du for space computation calculation, 2024-11-22T18:55:22,232 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/001b4aa4-1a7f-6591-3cef-5770bc0658f1/cluster_d9590100-86d5-54e6-c888-90717c40c23c/data/data1/current/BP-1295690749-172.17.0.2-1732301720241/current, will proceed with Du for space computation calculation, 2024-11-22T18:55:22,281 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-22T18:55:22,343 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe327dc54c0d6ba7a with lease ID 0x1aaa22a054f1ec73: Processing first storage report for DS-60f41134-33f1-4b7b-898d-e456b006f505 from datanode DatanodeRegistration(127.0.0.1:32923, datanodeUuid=d7ea0a5a-d1bc-40cd-b1e0-3faca14101ae, infoPort=44297, infoSecurePort=0, ipcPort=43357, storageInfo=lv=-57;cid=testClusterID;nsid=1324134285;c=1732301720241) 2024-11-22T18:55:22,345 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe327dc54c0d6ba7a with lease ID 0x1aaa22a054f1ec73: from storage DS-60f41134-33f1-4b7b-898d-e456b006f505 node DatanodeRegistration(127.0.0.1:32923, datanodeUuid=d7ea0a5a-d1bc-40cd-b1e0-3faca14101ae, infoPort=44297, infoSecurePort=0, ipcPort=43357, storageInfo=lv=-57;cid=testClusterID;nsid=1324134285;c=1732301720241), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-22T18:55:22,345 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe327dc54c0d6ba7a with lease ID 0x1aaa22a054f1ec73: Processing first storage report for DS-18bc2760-6003-4f37-b417-30973ad25bfd from datanode DatanodeRegistration(127.0.0.1:32923, datanodeUuid=d7ea0a5a-d1bc-40cd-b1e0-3faca14101ae, infoPort=44297, infoSecurePort=0, ipcPort=43357, storageInfo=lv=-57;cid=testClusterID;nsid=1324134285;c=1732301720241) 2024-11-22T18:55:22,346 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe327dc54c0d6ba7a with lease ID 0x1aaa22a054f1ec73: from storage DS-18bc2760-6003-4f37-b417-30973ad25bfd node DatanodeRegistration(127.0.0.1:32923, datanodeUuid=d7ea0a5a-d1bc-40cd-b1e0-3faca14101ae, infoPort=44297, infoSecurePort=0, ipcPort=43357, storageInfo=lv=-57;cid=testClusterID;nsid=1324134285;c=1732301720241), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-22T18:55:22,432 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/001b4aa4-1a7f-6591-3cef-5770bc0658f1 2024-11-22T18:55:22,528 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/001b4aa4-1a7f-6591-3cef-5770bc0658f1/cluster_d9590100-86d5-54e6-c888-90717c40c23c/zookeeper_0, clientPort=62869, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/001b4aa4-1a7f-6591-3cef-5770bc0658f1/cluster_d9590100-86d5-54e6-c888-90717c40c23c/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/001b4aa4-1a7f-6591-3cef-5770bc0658f1/cluster_d9590100-86d5-54e6-c888-90717c40c23c/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-22T18:55:22,539 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=62869 2024-11-22T18:55:22,550 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-22T18:55:22,553 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-22T18:55:22,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741825_1001 (size=7) 2024-11-22T18:55:23,233 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875 with version=8 2024-11-22T18:55:23,234 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/hbase-staging 2024-11-22T18:55:23,336 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-22T18:55:23,610 INFO [Time-limited test {}] client.ConnectionUtils(128): master/d79ba0c344fb:0 server-side Connection retries=45 2024-11-22T18:55:23,622 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T18:55:23,623 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-22T18:55:23,628 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-22T18:55:23,628 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T18:55:23,629 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-22T18:55:23,794 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-22T18:55:23,858 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-22T18:55:23,869 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-22T18:55:23,873 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-22T18:55:23,901 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 29661 (auto-detected) 2024-11-22T18:55:23,902 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-22T18:55:23,921 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44109 2024-11-22T18:55:23,945 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:44109 connecting to ZooKeeper ensemble=127.0.0.1:62869 2024-11-22T18:55:23,980 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:441090x0, quorum=127.0.0.1:62869, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-22T18:55:23,983 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:44109-0x10141087e180000 connected 2024-11-22T18:55:24,012 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-22T18:55:24,015 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-22T18:55:24,025 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44109-0x10141087e180000, quorum=127.0.0.1:62869, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T18:55:24,029 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875, hbase.cluster.distributed=false 2024-11-22T18:55:24,058 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44109-0x10141087e180000, quorum=127.0.0.1:62869, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-22T18:55:24,064 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44109 2024-11-22T18:55:24,065 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44109 2024-11-22T18:55:24,072 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44109 2024-11-22T18:55:24,076 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44109 2024-11-22T18:55:24,084 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44109 2024-11-22T18:55:24,219 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/d79ba0c344fb:0 server-side Connection retries=45 2024-11-22T18:55:24,221 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T18:55:24,221 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-22T18:55:24,221 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-22T18:55:24,221 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T18:55:24,221 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-22T18:55:24,224 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-22T18:55:24,227 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-22T18:55:24,228 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43593 2024-11-22T18:55:24,230 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43593 connecting to ZooKeeper ensemble=127.0.0.1:62869 2024-11-22T18:55:24,231 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-22T18:55:24,235 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-22T18:55:24,242 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:435930x0, quorum=127.0.0.1:62869, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-22T18:55:24,243 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:435930x0, quorum=127.0.0.1:62869, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T18:55:24,248 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-22T18:55:24,253 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43593-0x10141087e180001 connected 2024-11-22T18:55:24,258 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-22T18:55:24,261 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43593-0x10141087e180001, quorum=127.0.0.1:62869, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-22T18:55:24,266 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43593-0x10141087e180001, quorum=127.0.0.1:62869, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-22T18:55:24,268 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43593 2024-11-22T18:55:24,269 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43593 2024-11-22T18:55:24,269 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43593 2024-11-22T18:55:24,270 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43593 2024-11-22T18:55:24,270 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43593 2024-11-22T18:55:24,289 DEBUG [M:0;d79ba0c344fb:44109 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;d79ba0c344fb:44109 2024-11-22T18:55:24,290 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/d79ba0c344fb,44109,1732301723398 2024-11-22T18:55:24,298 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43593-0x10141087e180001, quorum=127.0.0.1:62869, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T18:55:24,298 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44109-0x10141087e180000, quorum=127.0.0.1:62869, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T18:55:24,300 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44109-0x10141087e180000, quorum=127.0.0.1:62869, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/d79ba0c344fb,44109,1732301723398 2024-11-22T18:55:24,323 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43593-0x10141087e180001, quorum=127.0.0.1:62869, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-22T18:55:24,323 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44109-0x10141087e180000, quorum=127.0.0.1:62869, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T18:55:24,324 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43593-0x10141087e180001, quorum=127.0.0.1:62869, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T18:55:24,325 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44109-0x10141087e180000, quorum=127.0.0.1:62869, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-22T18:55:24,326 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/d79ba0c344fb,44109,1732301723398 from backup master directory 2024-11-22T18:55:24,329 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44109-0x10141087e180000, quorum=127.0.0.1:62869, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/d79ba0c344fb,44109,1732301723398 2024-11-22T18:55:24,329 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43593-0x10141087e180001, quorum=127.0.0.1:62869, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T18:55:24,329 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44109-0x10141087e180000, quorum=127.0.0.1:62869, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T18:55:24,330 WARN [master/d79ba0c344fb:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-22T18:55:24,330 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=d79ba0c344fb,44109,1732301723398 2024-11-22T18:55:24,333 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-22T18:55:24,335 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-22T18:55:24,400 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/hbase.id] with ID: 058c0dbe-7061-419a-804d-605a1c1b6341 2024-11-22T18:55:24,400 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/.tmp/hbase.id 2024-11-22T18:55:24,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741826_1002 (size=42) 2024-11-22T18:55:24,813 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/.tmp/hbase.id]:[hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/hbase.id] 2024-11-22T18:55:24,859 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-22T18:55:24,865 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-22T18:55:24,888 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 21ms. 2024-11-22T18:55:24,892 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44109-0x10141087e180000, quorum=127.0.0.1:62869, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T18:55:24,892 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43593-0x10141087e180001, quorum=127.0.0.1:62869, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T18:55:24,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741827_1003 (size=196) 2024-11-22T18:55:25,326 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T18:55:25,329 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-22T18:55:25,346 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:55:25,351 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-22T18:55:25,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741828_1004 (size=1189) 2024-11-22T18:55:25,807 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/MasterData/data/master/store 2024-11-22T18:55:25,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741829_1005 (size=34) 2024-11-22T18:55:25,834 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-22T18:55:25,837 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T18:55:25,838 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-22T18:55:25,838 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T18:55:25,838 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T18:55:25,840 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-22T18:55:25,840 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T18:55:25,840 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T18:55:25,841 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732301725838Disabling compacts and flushes for region at 1732301725838Disabling writes for close at 1732301725840 (+2 ms)Writing region close event to WAL at 1732301725840Closed at 1732301725840 2024-11-22T18:55:25,844 WARN [master/d79ba0c344fb:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/MasterData/data/master/store/.initializing 2024-11-22T18:55:25,844 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/MasterData/WALs/d79ba0c344fb,44109,1732301723398 2024-11-22T18:55:25,854 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-22T18:55:25,872 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d79ba0c344fb%2C44109%2C1732301723398, suffix=, logDir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/MasterData/WALs/d79ba0c344fb,44109,1732301723398, archiveDir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/MasterData/oldWALs, maxLogs=10 2024-11-22T18:55:25,899 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/MasterData/WALs/d79ba0c344fb,44109,1732301723398/d79ba0c344fb%2C44109%2C1732301723398.1732301725877, exclude list is [], retry=0 2024-11-22T18:55:25,918 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32923,DS-60f41134-33f1-4b7b-898d-e456b006f505,DISK] 2024-11-22T18:55:25,921 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-22T18:55:25,962 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/MasterData/WALs/d79ba0c344fb,44109,1732301723398/d79ba0c344fb%2C44109%2C1732301723398.1732301725877 2024-11-22T18:55:25,964 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44297:44297)] 2024-11-22T18:55:25,964 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-22T18:55:25,965 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T18:55:25,969 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T18:55:25,970 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T18:55:26,012 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-22T18:55:26,041 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-22T18:55:26,046 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:55:26,049 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T18:55:26,049 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-22T18:55:26,053 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-22T18:55:26,053 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:55:26,054 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T18:55:26,054 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-22T18:55:26,057 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-22T18:55:26,057 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:55:26,059 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T18:55:26,059 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-22T18:55:26,062 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-22T18:55:26,062 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:55:26,063 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T18:55:26,063 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T18:55:26,067 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-22T18:55:26,068 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-22T18:55:26,073 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T18:55:26,074 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T18:55:26,077 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-22T18:55:26,081 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T18:55:26,086 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T18:55:26,087 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64889646, jitterRate=-0.03306892514228821}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-22T18:55:26,092 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732301725986Initializing all the Stores at 1732301725988 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732301725989 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732301725990 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732301725990Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732301725990Cleaning up temporary data from old regions at 1732301726074 (+84 ms)Region opened successfully at 1732301726092 (+18 ms) 2024-11-22T18:55:26,093 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-22T18:55:26,132 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d757ebf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d79ba0c344fb/172.17.0.2:0 2024-11-22T18:55:26,169 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-22T18:55:26,182 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-22T18:55:26,182 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-22T18:55:26,186 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T18:55:26,187 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-22T18:55:26,193 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-22T18:55:26,193 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-22T18:55:26,228 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-22T18:55:26,239 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44109-0x10141087e180000, quorum=127.0.0.1:62869, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-22T18:55:26,243 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-22T18:55:26,246 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-22T18:55:26,247 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44109-0x10141087e180000, quorum=127.0.0.1:62869, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-22T18:55:26,249 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-22T18:55:26,251 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-22T18:55:26,255 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44109-0x10141087e180000, quorum=127.0.0.1:62869, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-22T18:55:26,257 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-22T18:55:26,258 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44109-0x10141087e180000, quorum=127.0.0.1:62869, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-22T18:55:26,260 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-22T18:55:26,279 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44109-0x10141087e180000, quorum=127.0.0.1:62869, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-22T18:55:26,285 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-22T18:55:26,289 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44109-0x10141087e180000, quorum=127.0.0.1:62869, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-22T18:55:26,289 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43593-0x10141087e180001, quorum=127.0.0.1:62869, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-22T18:55:26,289 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44109-0x10141087e180000, quorum=127.0.0.1:62869, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T18:55:26,289 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43593-0x10141087e180001, quorum=127.0.0.1:62869, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T18:55:26,292 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=d79ba0c344fb,44109,1732301723398, sessionid=0x10141087e180000, setting cluster-up flag (Was=false) 2024-11-22T18:55:26,304 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44109-0x10141087e180000, quorum=127.0.0.1:62869, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T18:55:26,304 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43593-0x10141087e180001, quorum=127.0.0.1:62869, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T18:55:26,312 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-22T18:55:26,314 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d79ba0c344fb,44109,1732301723398 2024-11-22T18:55:26,320 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43593-0x10141087e180001, quorum=127.0.0.1:62869, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T18:55:26,320 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44109-0x10141087e180000, quorum=127.0.0.1:62869, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T18:55:26,326 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-22T18:55:26,328 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d79ba0c344fb,44109,1732301723398 2024-11-22T18:55:26,335 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-22T18:55:26,377 INFO [RS:0;d79ba0c344fb:43593 {}] regionserver.HRegionServer(746): ClusterId : 058c0dbe-7061-419a-804d-605a1c1b6341 2024-11-22T18:55:26,381 DEBUG [RS:0;d79ba0c344fb:43593 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-22T18:55:26,388 DEBUG [RS:0;d79ba0c344fb:43593 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-22T18:55:26,388 DEBUG [RS:0;d79ba0c344fb:43593 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-22T18:55:26,391 DEBUG [RS:0;d79ba0c344fb:43593 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-22T18:55:26,391 DEBUG [RS:0;d79ba0c344fb:43593 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c0921f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d79ba0c344fb/172.17.0.2:0 2024-11-22T18:55:26,407 DEBUG [RS:0;d79ba0c344fb:43593 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;d79ba0c344fb:43593 2024-11-22T18:55:26,411 INFO [RS:0;d79ba0c344fb:43593 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-22T18:55:26,411 INFO [RS:0;d79ba0c344fb:43593 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-22T18:55:26,411 DEBUG [RS:0;d79ba0c344fb:43593 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-22T18:55:26,414 INFO [RS:0;d79ba0c344fb:43593 {}] regionserver.HRegionServer(2659): reportForDuty to master=d79ba0c344fb,44109,1732301723398 with port=43593, startcode=1732301724173 2024-11-22T18:55:26,413 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T18:55:26,423 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-22T18:55:26,427 DEBUG [RS:0;d79ba0c344fb:43593 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-22T18:55:26,430 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-22T18:55:26,437 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: d79ba0c344fb,44109,1732301723398 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-22T18:55:26,446 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/d79ba0c344fb:0, corePoolSize=5, maxPoolSize=5 2024-11-22T18:55:26,446 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/d79ba0c344fb:0, corePoolSize=5, maxPoolSize=5 2024-11-22T18:55:26,446 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/d79ba0c344fb:0, corePoolSize=5, maxPoolSize=5 2024-11-22T18:55:26,447 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/d79ba0c344fb:0, corePoolSize=5, maxPoolSize=5 2024-11-22T18:55:26,447 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/d79ba0c344fb:0, corePoolSize=10, maxPoolSize=10 2024-11-22T18:55:26,447 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/d79ba0c344fb:0, corePoolSize=1, maxPoolSize=1 2024-11-22T18:55:26,447 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/d79ba0c344fb:0, corePoolSize=2, maxPoolSize=2 2024-11-22T18:55:26,447 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/d79ba0c344fb:0, corePoolSize=1, maxPoolSize=1 2024-11-22T18:55:26,460 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732301756460 2024-11-22T18:55:26,462 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-22T18:55:26,464 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-22T18:55:26,464 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-22T18:55:26,465 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-22T18:55:26,467 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34779, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-22T18:55:26,469 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-22T18:55:26,469 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-22T18:55:26,470 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-22T18:55:26,470 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-22T18:55:26,473 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:55:26,473 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-22T18:55:26,475 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44109 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:26,482 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-22T18:55:26,488 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-22T18:55:26,489 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-22T18:55:26,489 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-22T18:55:26,496 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-22T18:55:26,497 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-22T18:55:26,500 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/d79ba0c344fb:0:becomeActiveMaster-HFileCleaner.large.0-1732301726498,5,FailOnTimeoutGroup] 2024-11-22T18:55:26,502 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/d79ba0c344fb:0:becomeActiveMaster-HFileCleaner.small.0-1732301726500,5,FailOnTimeoutGroup] 2024-11-22T18:55:26,502 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-22T18:55:26,502 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-22T18:55:26,504 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-22T18:55:26,504 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-22T18:55:26,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741831_1007 (size=1321) 2024-11-22T18:55:26,509 DEBUG [RS:0;d79ba0c344fb:43593 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-22T18:55:26,509 WARN [RS:0;d79ba0c344fb:43593 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-22T18:55:26,611 INFO [RS:0;d79ba0c344fb:43593 {}] regionserver.HRegionServer(2659): reportForDuty to master=d79ba0c344fb,44109,1732301723398 with port=43593, startcode=1732301724173 2024-11-22T18:55:26,613 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44109 {}] master.ServerManager(363): Checking decommissioned status of RegionServer d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:26,616 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44109 {}] master.ServerManager(517): Registering regionserver=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:26,624 DEBUG [RS:0;d79ba0c344fb:43593 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875 2024-11-22T18:55:26,624 DEBUG [RS:0;d79ba0c344fb:43593 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:41517 2024-11-22T18:55:26,624 DEBUG [RS:0;d79ba0c344fb:43593 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-22T18:55:26,628 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44109-0x10141087e180000, quorum=127.0.0.1:62869, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-22T18:55:26,629 DEBUG [RS:0;d79ba0c344fb:43593 {}] zookeeper.ZKUtil(111): regionserver:43593-0x10141087e180001, quorum=127.0.0.1:62869, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:26,629 WARN [RS:0;d79ba0c344fb:43593 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-22T18:55:26,629 INFO [RS:0;d79ba0c344fb:43593 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-22T18:55:26,630 DEBUG [RS:0;d79ba0c344fb:43593 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/WALs/d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:26,631 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [d79ba0c344fb,43593,1732301724173] 2024-11-22T18:55:26,657 INFO [RS:0;d79ba0c344fb:43593 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-22T18:55:26,673 INFO [RS:0;d79ba0c344fb:43593 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-22T18:55:26,678 INFO [RS:0;d79ba0c344fb:43593 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-22T18:55:26,679 INFO [RS:0;d79ba0c344fb:43593 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T18:55:26,680 INFO [RS:0;d79ba0c344fb:43593 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-22T18:55:26,686 INFO [RS:0;d79ba0c344fb:43593 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-22T18:55:26,687 INFO [RS:0;d79ba0c344fb:43593 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-22T18:55:26,688 DEBUG [RS:0;d79ba0c344fb:43593 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/d79ba0c344fb:0, corePoolSize=1, maxPoolSize=1 2024-11-22T18:55:26,688 DEBUG [RS:0;d79ba0c344fb:43593 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/d79ba0c344fb:0, corePoolSize=1, maxPoolSize=1 2024-11-22T18:55:26,688 DEBUG [RS:0;d79ba0c344fb:43593 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/d79ba0c344fb:0, corePoolSize=1, maxPoolSize=1 2024-11-22T18:55:26,688 DEBUG [RS:0;d79ba0c344fb:43593 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/d79ba0c344fb:0, corePoolSize=1, maxPoolSize=1 2024-11-22T18:55:26,688 DEBUG [RS:0;d79ba0c344fb:43593 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/d79ba0c344fb:0, corePoolSize=1, maxPoolSize=1 2024-11-22T18:55:26,689 DEBUG [RS:0;d79ba0c344fb:43593 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/d79ba0c344fb:0, corePoolSize=2, maxPoolSize=2 2024-11-22T18:55:26,689 DEBUG [RS:0;d79ba0c344fb:43593 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/d79ba0c344fb:0, corePoolSize=1, maxPoolSize=1 2024-11-22T18:55:26,689 DEBUG [RS:0;d79ba0c344fb:43593 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/d79ba0c344fb:0, corePoolSize=1, maxPoolSize=1 2024-11-22T18:55:26,689 DEBUG [RS:0;d79ba0c344fb:43593 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/d79ba0c344fb:0, corePoolSize=1, maxPoolSize=1 2024-11-22T18:55:26,689 DEBUG [RS:0;d79ba0c344fb:43593 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/d79ba0c344fb:0, corePoolSize=1, maxPoolSize=1 2024-11-22T18:55:26,690 DEBUG [RS:0;d79ba0c344fb:43593 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/d79ba0c344fb:0, corePoolSize=1, maxPoolSize=1 2024-11-22T18:55:26,690 DEBUG [RS:0;d79ba0c344fb:43593 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/d79ba0c344fb:0, corePoolSize=1, maxPoolSize=1 2024-11-22T18:55:26,690 DEBUG [RS:0;d79ba0c344fb:43593 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/d79ba0c344fb:0, corePoolSize=3, maxPoolSize=3 2024-11-22T18:55:26,690 DEBUG [RS:0;d79ba0c344fb:43593 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0, corePoolSize=3, maxPoolSize=3 2024-11-22T18:55:26,691 INFO [RS:0;d79ba0c344fb:43593 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-22T18:55:26,691 INFO [RS:0;d79ba0c344fb:43593 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-22T18:55:26,691 INFO [RS:0;d79ba0c344fb:43593 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T18:55:26,692 INFO [RS:0;d79ba0c344fb:43593 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-22T18:55:26,692 INFO [RS:0;d79ba0c344fb:43593 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-22T18:55:26,692 INFO [RS:0;d79ba0c344fb:43593 {}] hbase.ChoreService(168): Chore ScheduledChore name=d79ba0c344fb,43593,1732301724173-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-22T18:55:26,713 INFO [RS:0;d79ba0c344fb:43593 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-22T18:55:26,715 INFO [RS:0;d79ba0c344fb:43593 {}] hbase.ChoreService(168): Chore ScheduledChore name=d79ba0c344fb,43593,1732301724173-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T18:55:26,715 INFO [RS:0;d79ba0c344fb:43593 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T18:55:26,715 INFO [RS:0;d79ba0c344fb:43593 {}] regionserver.Replication(171): d79ba0c344fb,43593,1732301724173 started 2024-11-22T18:55:26,735 INFO [RS:0;d79ba0c344fb:43593 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T18:55:26,735 INFO [RS:0;d79ba0c344fb:43593 {}] regionserver.HRegionServer(1482): Serving as d79ba0c344fb,43593,1732301724173, RpcServer on d79ba0c344fb/172.17.0.2:43593, sessionid=0x10141087e180001 2024-11-22T18:55:26,736 DEBUG [RS:0;d79ba0c344fb:43593 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-22T18:55:26,736 DEBUG [RS:0;d79ba0c344fb:43593 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:26,737 DEBUG [RS:0;d79ba0c344fb:43593 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd79ba0c344fb,43593,1732301724173' 2024-11-22T18:55:26,737 DEBUG [RS:0;d79ba0c344fb:43593 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-22T18:55:26,738 DEBUG [RS:0;d79ba0c344fb:43593 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-22T18:55:26,739 DEBUG [RS:0;d79ba0c344fb:43593 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-22T18:55:26,739 DEBUG [RS:0;d79ba0c344fb:43593 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-22T18:55:26,739 DEBUG [RS:0;d79ba0c344fb:43593 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:26,739 DEBUG [RS:0;d79ba0c344fb:43593 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd79ba0c344fb,43593,1732301724173' 2024-11-22T18:55:26,739 DEBUG [RS:0;d79ba0c344fb:43593 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-22T18:55:26,740 DEBUG [RS:0;d79ba0c344fb:43593 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-22T18:55:26,740 DEBUG [RS:0;d79ba0c344fb:43593 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-22T18:55:26,741 INFO [RS:0;d79ba0c344fb:43593 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-22T18:55:26,741 INFO [RS:0;d79ba0c344fb:43593 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-22T18:55:26,846 INFO [RS:0;d79ba0c344fb:43593 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-22T18:55:26,850 INFO [RS:0;d79ba0c344fb:43593 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d79ba0c344fb%2C43593%2C1732301724173, suffix=, logDir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/WALs/d79ba0c344fb,43593,1732301724173, archiveDir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/oldWALs, maxLogs=32 2024-11-22T18:55:26,869 DEBUG [RS:0;d79ba0c344fb:43593 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/WALs/d79ba0c344fb,43593,1732301724173/d79ba0c344fb%2C43593%2C1732301724173.1732301726853, exclude list is [], retry=0 2024-11-22T18:55:26,874 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32923,DS-60f41134-33f1-4b7b-898d-e456b006f505,DISK] 2024-11-22T18:55:26,877 INFO [RS:0;d79ba0c344fb:43593 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/WALs/d79ba0c344fb,43593,1732301724173/d79ba0c344fb%2C43593%2C1732301724173.1732301726853 2024-11-22T18:55:26,878 DEBUG [RS:0;d79ba0c344fb:43593 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44297:44297)] 2024-11-22T18:55:26,907 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-22T18:55:26,908 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875 2024-11-22T18:55:26,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741833_1009 (size=32) 2024-11-22T18:55:27,320 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T18:55:27,324 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-22T18:55:27,326 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-22T18:55:27,327 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:55:27,327 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T18:55:27,328 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-22T18:55:27,330 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-22T18:55:27,330 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:55:27,331 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T18:55:27,331 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-22T18:55:27,334 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-22T18:55:27,334 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:55:27,335 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T18:55:27,335 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-22T18:55:27,338 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-22T18:55:27,338 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:55:27,339 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T18:55:27,339 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-22T18:55:27,341 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/hbase/meta/1588230740 2024-11-22T18:55:27,342 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/hbase/meta/1588230740 2024-11-22T18:55:27,345 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-22T18:55:27,346 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-22T18:55:27,346 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-22T18:55:27,349 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-22T18:55:27,353 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T18:55:27,354 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59893185, jitterRate=-0.10752199590206146}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-22T18:55:27,357 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732301727321Initializing all the Stores at 1732301727323 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732301727323Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732301727324 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732301727324Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732301727324Cleaning up temporary data from old regions at 1732301727346 (+22 ms)Region opened successfully at 1732301727357 (+11 ms) 2024-11-22T18:55:27,357 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-22T18:55:27,357 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-22T18:55:27,357 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-22T18:55:27,357 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-22T18:55:27,357 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-22T18:55:27,359 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-22T18:55:27,359 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732301727357Disabling compacts and flushes for region at 1732301727357Disabling writes for close at 1732301727357Writing region close event to WAL at 1732301727358 (+1 ms)Closed at 1732301727359 (+1 ms) 2024-11-22T18:55:27,363 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-22T18:55:27,363 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-22T18:55:27,370 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-22T18:55:27,379 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-22T18:55:27,381 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-22T18:55:27,535 DEBUG [d79ba0c344fb:44109 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-22T18:55:27,550 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:27,558 INFO [PEWorker-2 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d79ba0c344fb,43593,1732301724173, state=OPENING 2024-11-22T18:55:27,563 DEBUG [PEWorker-2 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-22T18:55:27,566 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43593-0x10141087e180001, quorum=127.0.0.1:62869, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T18:55:27,566 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44109-0x10141087e180000, quorum=127.0.0.1:62869, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T18:55:27,567 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T18:55:27,567 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T18:55:27,569 DEBUG [PEWorker-2 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-22T18:55:27,571 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=d79ba0c344fb,43593,1732301724173}] 2024-11-22T18:55:27,749 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-22T18:55:27,752 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56917, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-22T18:55:27,764 INFO [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-22T18:55:27,765 INFO [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-22T18:55:27,765 INFO [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-22T18:55:27,769 INFO [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d79ba0c344fb%2C43593%2C1732301724173.meta, suffix=.meta, logDir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/WALs/d79ba0c344fb,43593,1732301724173, archiveDir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/oldWALs, maxLogs=32 2024-11-22T18:55:27,787 DEBUG [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/WALs/d79ba0c344fb,43593,1732301724173/d79ba0c344fb%2C43593%2C1732301724173.meta.1732301727771.meta, exclude list is [], retry=0 2024-11-22T18:55:27,791 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32923,DS-60f41134-33f1-4b7b-898d-e456b006f505,DISK] 2024-11-22T18:55:27,797 INFO [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/WALs/d79ba0c344fb,43593,1732301724173/d79ba0c344fb%2C43593%2C1732301724173.meta.1732301727771.meta 2024-11-22T18:55:27,800 DEBUG [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44297:44297)] 2024-11-22T18:55:27,801 DEBUG [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-22T18:55:27,802 DEBUG [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-22T18:55:27,806 DEBUG [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-22T18:55:27,811 INFO [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-22T18:55:27,816 DEBUG [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-22T18:55:27,817 DEBUG [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T18:55:27,817 DEBUG [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-22T18:55:27,817 DEBUG [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-22T18:55:27,821 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-22T18:55:27,823 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-22T18:55:27,823 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:55:27,825 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T18:55:27,825 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-22T18:55:27,827 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-22T18:55:27,827 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:55:27,828 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T18:55:27,828 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-22T18:55:27,830 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-22T18:55:27,830 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:55:27,831 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T18:55:27,831 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-22T18:55:27,833 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-22T18:55:27,833 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:55:27,834 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T18:55:27,834 DEBUG [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-22T18:55:27,836 DEBUG [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/hbase/meta/1588230740 2024-11-22T18:55:27,839 DEBUG [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/hbase/meta/1588230740 2024-11-22T18:55:27,842 DEBUG [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-22T18:55:27,842 DEBUG [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-22T18:55:27,843 DEBUG [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-22T18:55:27,846 DEBUG [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-22T18:55:27,848 INFO [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66399531, jitterRate=-0.01056988537311554}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-22T18:55:27,848 DEBUG [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-22T18:55:27,850 DEBUG [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732301727818Writing region info on filesystem at 1732301727818Initializing all the Stores at 1732301727820 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732301727820Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732301727821 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732301727821Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732301727821Cleaning up temporary data from old regions at 1732301727842 (+21 ms)Running coprocessor post-open hooks at 1732301727848 (+6 ms)Region opened successfully at 1732301727850 (+2 ms) 2024-11-22T18:55:27,859 INFO [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732301727738 2024-11-22T18:55:27,872 DEBUG [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-22T18:55:27,872 INFO [RS_OPEN_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-22T18:55:27,875 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:27,877 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d79ba0c344fb,43593,1732301724173, state=OPEN 2024-11-22T18:55:27,884 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44109-0x10141087e180000, quorum=127.0.0.1:62869, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T18:55:27,884 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43593-0x10141087e180001, quorum=127.0.0.1:62869, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T18:55:27,884 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T18:55:27,884 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T18:55:27,885 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:27,892 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-22T18:55:27,894 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d79ba0c344fb,43593,1732301724173 in 315 msec 2024-11-22T18:55:27,899 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-22T18:55:27,899 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 525 msec 2024-11-22T18:55:27,901 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-22T18:55:27,901 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-22T18:55:27,924 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T18:55:27,925 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d79ba0c344fb,43593,1732301724173, seqNum=-1] 2024-11-22T18:55:27,950 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T18:55:27,952 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53693, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T18:55:27,976 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.6100 sec 2024-11-22T18:55:27,976 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732301727976, completionTime=-1 2024-11-22T18:55:27,979 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-22T18:55:27,979 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T18:55:28,008 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-22T18:55:28,009 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732301788009 2024-11-22T18:55:28,009 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732301848009 2024-11-22T18:55:28,009 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 29 msec 2024-11-22T18:55:28,011 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d79ba0c344fb,44109,1732301723398-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T18:55:28,011 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d79ba0c344fb,44109,1732301723398-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T18:55:28,012 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d79ba0c344fb,44109,1732301723398-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T18:55:28,013 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-d79ba0c344fb:44109, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T18:55:28,014 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-22T18:55:28,014 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-22T18:55:28,020 DEBUG [master/d79ba0c344fb:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-22T18:55:28,044 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.712sec 2024-11-22T18:55:28,045 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-22T18:55:28,046 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-22T18:55:28,047 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-22T18:55:28,048 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-22T18:55:28,048 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-22T18:55:28,049 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d79ba0c344fb,44109,1732301723398-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-22T18:55:28,049 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d79ba0c344fb,44109,1732301723398-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-22T18:55:28,058 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-22T18:55:28,059 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-22T18:55:28,060 INFO [master/d79ba0c344fb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d79ba0c344fb,44109,1732301723398-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T18:55:28,088 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@343aff6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:55:28,111 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-22T18:55:28,112 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-22T18:55:28,116 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d79ba0c344fb,44109,-1 for getting cluster id 2024-11-22T18:55:28,119 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T18:55:28,129 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '058c0dbe-7061-419a-804d-605a1c1b6341' 2024-11-22T18:55:28,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T18:55:28,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "058c0dbe-7061-419a-804d-605a1c1b6341" 2024-11-22T18:55:28,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b5c7a62, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:55:28,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d79ba0c344fb,44109,-1] 2024-11-22T18:55:28,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T18:55:28,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:55:28,146 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45258, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T18:55:28,149 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32372268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:55:28,150 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T18:55:28,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d79ba0c344fb,43593,1732301724173, seqNum=-1] 2024-11-22T18:55:28,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T18:55:28,161 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40702, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T18:55:28,188 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=d79ba0c344fb,44109,1732301723398 2024-11-22T18:55:28,204 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=265, ProcessCount=11, AvailableMemoryMB=7688 2024-11-22T18:55:28,214 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-22T18:55:28,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is d79ba0c344fb,44109,1732301723398 2024-11-22T18:55:28,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@35874ff9 2024-11-22T18:55:28,223 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-22T18:55:28,226 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45272, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-22T18:55:28,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-22T18:55:28,233 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T18:55:28,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-22T18:55:28,244 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-22T18:55:28,247 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-22T18:55:28,247 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:55:28,250 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-22T18:55:28,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T18:55:28,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741835_1011 (size=959) 2024-11-22T18:55:28,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T18:55:28,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T18:55:28,672 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875 2024-11-22T18:55:28,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741836_1012 (size=53) 2024-11-22T18:55:28,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T18:55:29,084 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T18:55:29,084 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 0fc957929e5be1c291085a12bd8cf851, disabling compactions & flushes 2024-11-22T18:55:29,084 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:29,084 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:29,085 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. after waiting 0 ms 2024-11-22T18:55:29,085 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:29,085 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:29,085 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0fc957929e5be1c291085a12bd8cf851: Waiting for close lock at 1732301729084Disabling compacts and flushes for region at 1732301729084Disabling writes for close at 1732301729085 (+1 ms)Writing region close event to WAL at 1732301729085Closed at 1732301729085 2024-11-22T18:55:29,088 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-22T18:55:29,094 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732301729088"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732301729088"}]},"ts":"1732301729088"} 2024-11-22T18:55:29,100 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-22T18:55:29,102 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-22T18:55:29,105 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732301729102"}]},"ts":"1732301729102"} 2024-11-22T18:55:29,109 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-22T18:55:29,111 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0fc957929e5be1c291085a12bd8cf851, ASSIGN}] 2024-11-22T18:55:29,114 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0fc957929e5be1c291085a12bd8cf851, ASSIGN 2024-11-22T18:55:29,115 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0fc957929e5be1c291085a12bd8cf851, ASSIGN; state=OFFLINE, location=d79ba0c344fb,43593,1732301724173; forceNewPlan=false, retain=false 2024-11-22T18:55:29,267 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=0fc957929e5be1c291085a12bd8cf851, regionState=OPENING, regionLocation=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:29,272 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0fc957929e5be1c291085a12bd8cf851, ASSIGN because future has completed 2024-11-22T18:55:29,273 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173}] 2024-11-22T18:55:29,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T18:55:29,435 INFO [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:29,435 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} 2024-11-22T18:55:29,435 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 0fc957929e5be1c291085a12bd8cf851 2024-11-22T18:55:29,436 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T18:55:29,436 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 0fc957929e5be1c291085a12bd8cf851 2024-11-22T18:55:29,436 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 0fc957929e5be1c291085a12bd8cf851 2024-11-22T18:55:29,438 INFO [StoreOpener-0fc957929e5be1c291085a12bd8cf851-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 0fc957929e5be1c291085a12bd8cf851 2024-11-22T18:55:29,441 INFO [StoreOpener-0fc957929e5be1c291085a12bd8cf851-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0fc957929e5be1c291085a12bd8cf851 columnFamilyName A 2024-11-22T18:55:29,441 DEBUG [StoreOpener-0fc957929e5be1c291085a12bd8cf851-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:55:29,442 INFO [StoreOpener-0fc957929e5be1c291085a12bd8cf851-1 {}] regionserver.HStore(327): Store=0fc957929e5be1c291085a12bd8cf851/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T18:55:29,443 INFO [StoreOpener-0fc957929e5be1c291085a12bd8cf851-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 0fc957929e5be1c291085a12bd8cf851 2024-11-22T18:55:29,445 INFO [StoreOpener-0fc957929e5be1c291085a12bd8cf851-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0fc957929e5be1c291085a12bd8cf851 columnFamilyName B 2024-11-22T18:55:29,445 DEBUG [StoreOpener-0fc957929e5be1c291085a12bd8cf851-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:55:29,446 INFO [StoreOpener-0fc957929e5be1c291085a12bd8cf851-1 {}] regionserver.HStore(327): Store=0fc957929e5be1c291085a12bd8cf851/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T18:55:29,447 INFO [StoreOpener-0fc957929e5be1c291085a12bd8cf851-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 0fc957929e5be1c291085a12bd8cf851 2024-11-22T18:55:29,449 INFO [StoreOpener-0fc957929e5be1c291085a12bd8cf851-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0fc957929e5be1c291085a12bd8cf851 columnFamilyName C 2024-11-22T18:55:29,449 DEBUG [StoreOpener-0fc957929e5be1c291085a12bd8cf851-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:55:29,450 INFO [StoreOpener-0fc957929e5be1c291085a12bd8cf851-1 {}] regionserver.HStore(327): Store=0fc957929e5be1c291085a12bd8cf851/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T18:55:29,450 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 0fc957929e5be1c291085a12bd8cf851 2024-11-22T18:55:29,451 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851 2024-11-22T18:55:29,452 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851 2024-11-22T18:55:29,454 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 0fc957929e5be1c291085a12bd8cf851 2024-11-22T18:55:29,454 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 0fc957929e5be1c291085a12bd8cf851 2024-11-22T18:55:29,455 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-22T18:55:29,458 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 0fc957929e5be1c291085a12bd8cf851 2024-11-22T18:55:29,462 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T18:55:29,463 INFO [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 0fc957929e5be1c291085a12bd8cf851; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60475547, jitterRate=-0.09884412586688995}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-22T18:55:29,463 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0fc957929e5be1c291085a12bd8cf851 2024-11-22T18:55:29,464 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 0fc957929e5be1c291085a12bd8cf851: Running coprocessor pre-open hook at 1732301729436Writing region info on filesystem at 1732301729436Initializing all the Stores at 1732301729438 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732301729438Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732301729438Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732301729438Cleaning up temporary data from old regions at 1732301729454 (+16 ms)Running coprocessor post-open hooks at 1732301729463 (+9 ms)Region opened successfully at 1732301729464 (+1 ms) 2024-11-22T18:55:29,467 INFO [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., pid=6, masterSystemTime=1732301729427 2024-11-22T18:55:29,471 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:29,471 INFO [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:29,472 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=0fc957929e5be1c291085a12bd8cf851, regionState=OPEN, openSeqNum=2, regionLocation=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:29,476 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 because future has completed 2024-11-22T18:55:29,482 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-22T18:55:29,482 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 in 205 msec 2024-11-22T18:55:29,486 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-22T18:55:29,487 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0fc957929e5be1c291085a12bd8cf851, ASSIGN in 371 msec 2024-11-22T18:55:29,489 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-22T18:55:29,489 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732301729489"}]},"ts":"1732301729489"} 2024-11-22T18:55:29,493 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-22T18:55:29,495 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-22T18:55:29,498 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2590 sec 2024-11-22T18:55:30,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T18:55:30,408 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-22T18:55:30,413 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad452c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:55:30,413 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d79ba0c344fb,44109,-1 for getting cluster id 2024-11-22T18:55:30,414 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T18:55:30,415 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '058c0dbe-7061-419a-804d-605a1c1b6341' 2024-11-22T18:55:30,416 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T18:55:30,416 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "058c0dbe-7061-419a-804d-605a1c1b6341" 2024-11-22T18:55:30,416 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46c7780, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:55:30,416 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d79ba0c344fb,44109,-1] 2024-11-22T18:55:30,416 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T18:55:30,417 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:55:30,418 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45294, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T18:55:30,420 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@89b31dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:55:30,420 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T18:55:30,422 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d79ba0c344fb,43593,1732301724173, seqNum=-1] 2024-11-22T18:55:30,422 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T18:55:30,424 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40718, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T18:55:30,426 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-22T18:55:30,427 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is d79ba0c344fb,44109,1732301723398 2024-11-22T18:55:30,427 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4e07ee28 2024-11-22T18:55:30,429 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-22T18:55:30,431 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45296, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-22T18:55:30,435 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19d419ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:55:30,435 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d79ba0c344fb,44109,-1 for getting cluster id 2024-11-22T18:55:30,435 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T18:55:30,436 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '058c0dbe-7061-419a-804d-605a1c1b6341' 2024-11-22T18:55:30,437 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T18:55:30,437 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "058c0dbe-7061-419a-804d-605a1c1b6341" 2024-11-22T18:55:30,437 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e1a1116, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:55:30,437 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d79ba0c344fb,44109,-1] 2024-11-22T18:55:30,437 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T18:55:30,438 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:55:30,439 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45312, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T18:55:30,440 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e59e9dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:55:30,441 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@130d4a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:55:30,441 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d79ba0c344fb,44109,-1 for getting cluster id 2024-11-22T18:55:30,441 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T18:55:30,443 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '058c0dbe-7061-419a-804d-605a1c1b6341' 2024-11-22T18:55:30,443 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T18:55:30,443 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "058c0dbe-7061-419a-804d-605a1c1b6341" 2024-11-22T18:55:30,444 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ab7eb50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:55:30,444 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d79ba0c344fb,44109,-1] 2024-11-22T18:55:30,444 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T18:55:30,444 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:55:30,446 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45326, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T18:55:30,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a9c1aaa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:55:30,447 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d35085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:55:30,447 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d79ba0c344fb,44109,-1 for getting cluster id 2024-11-22T18:55:30,448 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T18:55:30,449 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '058c0dbe-7061-419a-804d-605a1c1b6341' 2024-11-22T18:55:30,449 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T18:55:30,449 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "058c0dbe-7061-419a-804d-605a1c1b6341" 2024-11-22T18:55:30,450 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3937ace3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:55:30,450 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d79ba0c344fb,44109,-1] 2024-11-22T18:55:30,450 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T18:55:30,450 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:55:30,452 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45352, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T18:55:30,452 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30bed14c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:55:30,453 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fb4b46b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:55:30,453 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d79ba0c344fb,44109,-1 for getting cluster id 2024-11-22T18:55:30,453 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T18:55:30,456 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '058c0dbe-7061-419a-804d-605a1c1b6341' 2024-11-22T18:55:30,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T18:55:30,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "058c0dbe-7061-419a-804d-605a1c1b6341" 2024-11-22T18:55:30,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b130fb1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:55:30,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d79ba0c344fb,44109,-1] 2024-11-22T18:55:30,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T18:55:30,458 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:55:30,459 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45370, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T18:55:30,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ce4b195, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:55:30,461 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a1cce95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:55:30,461 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d79ba0c344fb,44109,-1 for getting cluster id 2024-11-22T18:55:30,461 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T18:55:30,462 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '058c0dbe-7061-419a-804d-605a1c1b6341' 2024-11-22T18:55:30,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T18:55:30,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "058c0dbe-7061-419a-804d-605a1c1b6341" 2024-11-22T18:55:30,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5805ae29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:55:30,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d79ba0c344fb,44109,-1] 2024-11-22T18:55:30,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T18:55:30,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:55:30,463 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45382, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T18:55:30,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d230d79, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:55:30,466 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3055dae7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:55:30,466 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d79ba0c344fb,44109,-1 for getting cluster id 2024-11-22T18:55:30,466 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T18:55:30,467 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '058c0dbe-7061-419a-804d-605a1c1b6341' 2024-11-22T18:55:30,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T18:55:30,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "058c0dbe-7061-419a-804d-605a1c1b6341" 2024-11-22T18:55:30,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7af5f6a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:55:30,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d79ba0c344fb,44109,-1] 2024-11-22T18:55:30,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T18:55:30,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:55:30,469 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45400, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T18:55:30,470 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d37adaa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:55:30,470 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5238acdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:55:30,470 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d79ba0c344fb,44109,-1 for getting cluster id 2024-11-22T18:55:30,470 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T18:55:30,471 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '058c0dbe-7061-419a-804d-605a1c1b6341' 2024-11-22T18:55:30,471 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T18:55:30,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "058c0dbe-7061-419a-804d-605a1c1b6341" 2024-11-22T18:55:30,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3adce01e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:55:30,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d79ba0c344fb,44109,-1] 2024-11-22T18:55:30,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T18:55:30,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:55:30,473 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45416, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T18:55:30,474 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7256f4ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:55:30,475 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f4f2c18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:55:30,475 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d79ba0c344fb,44109,-1 for getting cluster id 2024-11-22T18:55:30,475 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T18:55:30,476 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '058c0dbe-7061-419a-804d-605a1c1b6341' 2024-11-22T18:55:30,476 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T18:55:30,476 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "058c0dbe-7061-419a-804d-605a1c1b6341" 2024-11-22T18:55:30,477 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a0dfe7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:55:30,477 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d79ba0c344fb,44109,-1] 2024-11-22T18:55:30,477 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T18:55:30,477 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:55:30,478 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45438, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T18:55:30,479 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28d04c0c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:55:30,479 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6643136e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:55:30,479 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d79ba0c344fb,44109,-1 for getting cluster id 2024-11-22T18:55:30,480 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T18:55:30,480 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '058c0dbe-7061-419a-804d-605a1c1b6341' 2024-11-22T18:55:30,481 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T18:55:30,481 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "058c0dbe-7061-419a-804d-605a1c1b6341" 2024-11-22T18:55:30,481 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55a56f43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:55:30,481 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d79ba0c344fb,44109,-1] 2024-11-22T18:55:30,482 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T18:55:30,482 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:55:30,483 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45464, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T18:55:30,483 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b2db193, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:55:30,489 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T18:55:30,489 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T18:55:30,489 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T18:55:30,489 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T18:55:30,490 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T18:55:30,491 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d79ba0c344fb,43593,1732301724173, seqNum=-1] 2024-11-22T18:55:30,491 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d79ba0c344fb,43593,1732301724173, seqNum=-1] 2024-11-22T18:55:30,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d79ba0c344fb,43593,1732301724173, seqNum=-1] 2024-11-22T18:55:30,491 DEBUG [Thread-147 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d79ba0c344fb,43593,1732301724173, seqNum=-1] 2024-11-22T18:55:30,491 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T18:55:30,491 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T18:55:30,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T18:55:30,491 DEBUG [Thread-147 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T18:55:30,492 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T18:55:30,492 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d79ba0c344fb,43593,1732301724173, seqNum=-1] 2024-11-22T18:55:30,493 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T18:55:30,493 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40756, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T18:55:30,493 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40734, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T18:55:30,493 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40746, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T18:55:30,493 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40758, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T18:55:30,494 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40764, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T18:55:30,494 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d79ba0c344fb,43593,1732301724173, seqNum=-1] 2024-11-22T18:55:30,495 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T18:55:30,496 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40772, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T18:55:30,497 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T18:55:30,497 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T18:55:30,498 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d79ba0c344fb,43593,1732301724173, seqNum=-1] 2024-11-22T18:55:30,499 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T18:55:30,500 DEBUG [Thread-151 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d79ba0c344fb,43593,1732301724173, seqNum=-1] 2024-11-22T18:55:30,500 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2] 2024-11-22T18:55:30,501 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40776, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T18:55:30,501 DEBUG [Thread-151 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T18:55:30,501 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2] 2024-11-22T18:55:30,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2] 2024-11-22T18:55:30,503 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40782, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T18:55:30,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2] 2024-11-22T18:55:30,507 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2] 2024-11-22T18:55:30,507 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2] 2024-11-22T18:55:30,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2] 2024-11-22T18:55:30,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2] 2024-11-22T18:55:30,513 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T18:55:30,514 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T18:55:30,514 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d79ba0c344fb,43593,1732301724173, seqNum=-1] 2024-11-22T18:55:30,515 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T18:55:30,516 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40790, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T18:55:30,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2] 2024-11-22T18:55:30,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-22T18:55:30,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-22T18:55:30,529 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T18:55:30,531 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T18:55:30,533 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T18:55:30,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on 0fc957929e5be1c291085a12bd8cf851 2024-11-22T18:55:30,582 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T18:55:30,592 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0fc957929e5be1c291085a12bd8cf851 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-22T18:55:30,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-22T18:55:30,693 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/530a42fe465c4de3aa16c654d0b2d64b is 50, key is test_row_0/A:col10/1732301730550/Put/seqid=0 2024-11-22T18:55:30,715 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-22T18:55:30,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741837_1013 (size=12001) 2024-11-22T18:55:30,728 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/530a42fe465c4de3aa16c654d0b2d64b 2024-11-22T18:55:30,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:30,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. as already flushing 2024-11-22T18:55:30,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:30,737 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:55:30,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:55:30,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:55:30,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:30,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:30,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40782 deadline: 1732301790751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:30,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40746 deadline: 1732301790754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:30,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:30,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1732301790757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:30,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:30,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40756 deadline: 1732301790758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:30,786 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:30,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:30,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:30,786 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:30,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:30,786 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:30,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:30,786 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:30,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40758 deadline: 1732301790777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:30,786 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:30,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:30,786 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:30,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:30,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:30,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:30,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:30,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:30,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-22T18:55:30,869 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/6616ca09ab9e456c9f71da3c13334460 is 50, key is test_row_0/B:col10/1732301730550/Put/seqid=0 2024-11-22T18:55:30,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741838_1014 (size=12001) 2024-11-22T18:55:30,902 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/6616ca09ab9e456c9f71da3c13334460 2024-11-22T18:55:30,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:30,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40756 deadline: 1732301790899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:30,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:30,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40746 deadline: 1732301790899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:30,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:30,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:30,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:30,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40758 deadline: 1732301790900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:30,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:30,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:30,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:30,907 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-22T18:55:30,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:30,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:30,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40782 deadline: 1732301790899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:30,908 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:30,908 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:30,908 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:30,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:30,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:30,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1732301790906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:30,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:30,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:30,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:30,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:30,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:30,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:30,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. as already flushing 2024-11-22T18:55:30,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:30,925 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:55:30,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:55:30,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:55:30,963 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/c45836aee3c643dfa7a9f228a0926a2b is 50, key is test_row_0/C:col10/1732301730550/Put/seqid=0 2024-11-22T18:55:31,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741839_1015 (size=12001) 2024-11-22T18:55:31,085 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-22T18:55:31,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:31,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. as already flushing 2024-11-22T18:55:31,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:31,105 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:55:31,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:55:31,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:55:31,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:31,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40756 deadline: 1732301791116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:31,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:31,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:31,122 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:31,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:31,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:31,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40746 deadline: 1732301791118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:31,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40758 deadline: 1732301791117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:31,124 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:31,124 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:31,124 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:31,124 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:31,124 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:31,125 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:31,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:31,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40782 deadline: 1732301791118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:31,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:31,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:31,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:31,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:31,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1732301791118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:31,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:31,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:31,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:31,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-22T18:55:31,261 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-22T18:55:31,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:31,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. as already flushing 2024-11-22T18:55:31,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:31,261 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:55:31,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:55:31,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:55:31,408 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/c45836aee3c643dfa7a9f228a0926a2b 2024-11-22T18:55:31,417 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-22T18:55:31,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:31,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. as already flushing 2024-11-22T18:55:31,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:31,418 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:55:31,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:55:31,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:55:31,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/530a42fe465c4de3aa16c654d0b2d64b as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/530a42fe465c4de3aa16c654d0b2d64b 2024-11-22T18:55:31,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:31,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40756 deadline: 1732301791427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:31,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:31,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:31,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:31,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:31,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:31,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40758 deadline: 1732301791437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:31,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40746 deadline: 1732301791437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:31,440 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:31,440 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/530a42fe465c4de3aa16c654d0b2d64b, entries=150, sequenceid=14, filesize=11.7 K 2024-11-22T18:55:31,440 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:31,440 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:31,440 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:31,440 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:31,440 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:31,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:31,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40782 deadline: 1732301791438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:31,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:31,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:31,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:31,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:31,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1732301791438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:31,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:31,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:31,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:31,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/6616ca09ab9e456c9f71da3c13334460 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/6616ca09ab9e456c9f71da3c13334460 2024-11-22T18:55:31,459 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/6616ca09ab9e456c9f71da3c13334460, entries=150, sequenceid=14, filesize=11.7 K 2024-11-22T18:55:31,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/c45836aee3c643dfa7a9f228a0926a2b as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/c45836aee3c643dfa7a9f228a0926a2b 2024-11-22T18:55:31,473 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/c45836aee3c643dfa7a9f228a0926a2b, entries=150, sequenceid=14, filesize=11.7 K 2024-11-22T18:55:31,481 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 0fc957929e5be1c291085a12bd8cf851 in 893ms, sequenceid=14, compaction requested=false 2024-11-22T18:55:31,486 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-22T18:55:31,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0fc957929e5be1c291085a12bd8cf851: 2024-11-22T18:55:31,573 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-22T18:55:31,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:31,574 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 0fc957929e5be1c291085a12bd8cf851 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-22T18:55:31,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/1f4e77d17cbe4deead62ce1933cb5be1 is 50, key is test_row_0/A:col10/1732301730772/Put/seqid=0 2024-11-22T18:55:31,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741840_1016 (size=12001) 2024-11-22T18:55:31,639 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/1f4e77d17cbe4deead62ce1933cb5be1 2024-11-22T18:55:31,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-22T18:55:31,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/5304e9a1622746cd996429ba8355d625 is 50, key is test_row_0/B:col10/1732301730772/Put/seqid=0 2024-11-22T18:55:31,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741841_1017 (size=12001) 2024-11-22T18:55:31,700 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/5304e9a1622746cd996429ba8355d625 2024-11-22T18:55:31,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/ffbe2484e7754ff089bb9b8da23c538d is 50, key is test_row_0/C:col10/1732301730772/Put/seqid=0 2024-11-22T18:55:31,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741842_1018 (size=12001) 2024-11-22T18:55:31,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. as already flushing 2024-11-22T18:55:31,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on 0fc957929e5be1c291085a12bd8cf851 2024-11-22T18:55:31,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:31,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1732301791975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:31,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:31,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:31,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:31,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:31,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40746 deadline: 1732301791977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:31,991 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:31,991 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:31,991 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:31,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:31,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40758 deadline: 1732301791984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:31,994 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:31,994 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:31,994 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:31,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:31,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40756 deadline: 1732301791989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:31,996 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:31,996 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:31,997 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:31,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:31,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40782 deadline: 1732301791989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:31,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:31,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:31,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:32,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:32,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1732301792097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:32,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:32,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:32,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:32,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:32,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40746 deadline: 1732301792097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:32,101 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:32,101 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:32,101 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:32,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:32,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40758 deadline: 1732301792097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:32,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:32,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:32,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:32,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:32,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40756 deadline: 1732301792106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:32,110 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:32,110 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:32,110 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:32,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:32,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40782 deadline: 1732301792107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:32,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:32,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:32,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:32,202 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/ffbe2484e7754ff089bb9b8da23c538d 2024-11-22T18:55:32,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/1f4e77d17cbe4deead62ce1933cb5be1 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/1f4e77d17cbe4deead62ce1933cb5be1 2024-11-22T18:55:32,238 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/1f4e77d17cbe4deead62ce1933cb5be1, entries=150, sequenceid=37, filesize=11.7 K 2024-11-22T18:55:32,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/5304e9a1622746cd996429ba8355d625 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/5304e9a1622746cd996429ba8355d625 2024-11-22T18:55:32,255 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/5304e9a1622746cd996429ba8355d625, entries=150, sequenceid=37, filesize=11.7 K 2024-11-22T18:55:32,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/ffbe2484e7754ff089bb9b8da23c538d as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/ffbe2484e7754ff089bb9b8da23c538d 2024-11-22T18:55:32,273 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/ffbe2484e7754ff089bb9b8da23c538d, entries=150, sequenceid=37, filesize=11.7 K 2024-11-22T18:55:32,275 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for 0fc957929e5be1c291085a12bd8cf851 in 701ms, sequenceid=37, compaction requested=false 2024-11-22T18:55:32,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 0fc957929e5be1c291085a12bd8cf851: 2024-11-22T18:55:32,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:32,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-22T18:55:32,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-22T18:55:32,282 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-22T18:55:32,282 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7450 sec 2024-11-22T18:55:32,290 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.7720 sec 2024-11-22T18:55:32,317 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T18:55:32,317 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0fc957929e5be1c291085a12bd8cf851 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-22T18:55:32,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on 0fc957929e5be1c291085a12bd8cf851 2024-11-22T18:55:32,333 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/68bc0ffc9d98419182f133302b83f612 is 50, key is test_row_0/A:col10/1732301731985/Put/seqid=0 2024-11-22T18:55:32,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741843_1019 (size=12001) 2024-11-22T18:55:32,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:32,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40756 deadline: 1732301792369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:32,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:32,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:32,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:32,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:32,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1732301792370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:32,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:32,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:32,381 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:32,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:32,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40758 deadline: 1732301792374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:32,383 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:32,383 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:32,383 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:32,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:32,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40746 deadline: 1732301792377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:32,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:32,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40782 deadline: 1732301792379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:32,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:32,386 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:32,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:32,386 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:32,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:32,386 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:32,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:32,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40756 deadline: 1732301792487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:32,492 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:32,492 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:32,492 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:32,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:32,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1732301792487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:32,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:32,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:32,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:32,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:32,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40758 deadline: 1732301792487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:32,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:32,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:32,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:32,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:32,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40782 deadline: 1732301792495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:32,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:32,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:32,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:32,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:32,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40746 deadline: 1732301792496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:32,501 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:32,501 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:32,501 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:32,650 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-22T18:55:32,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-22T18:55:32,686 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T18:55:32,687 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T18:55:32,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-22T18:55:32,691 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T18:55:32,693 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T18:55:32,693 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T18:55:32,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-22T18:55:32,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:32,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40756 deadline: 1732301792696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:32,705 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:32,706 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:32,706 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:32,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:32,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1732301792706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:32,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:32,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40782 deadline: 1732301792706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:32,712 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:32,712 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:32,712 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:32,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:32,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:32,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:32,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:32,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40746 deadline: 1732301792707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:32,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:32,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40758 deadline: 1732301792706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:32,713 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:32,713 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:32,713 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:32,713 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:32,713 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:32,714 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:32,755 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/68bc0ffc9d98419182f133302b83f612 2024-11-22T18:55:32,784 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/03902c96c6c544109a2f6077ea555546 is 50, key is test_row_0/B:col10/1732301731985/Put/seqid=0 2024-11-22T18:55:32,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741844_1020 (size=12001) 2024-11-22T18:55:32,804 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/03902c96c6c544109a2f6077ea555546 2024-11-22T18:55:32,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-22T18:55:32,833 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/9c71c5ce5d5f4bdbabb6d078ac2814c6 is 50, key is test_row_0/C:col10/1732301731985/Put/seqid=0 2024-11-22T18:55:32,849 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-22T18:55:32,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:32,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. as already flushing 2024-11-22T18:55:32,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:32,850 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:55:32,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:55:32,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:55:32,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741845_1021 (size=12001) 2024-11-22T18:55:32,859 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/9c71c5ce5d5f4bdbabb6d078ac2814c6 2024-11-22T18:55:32,879 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/68bc0ffc9d98419182f133302b83f612 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/68bc0ffc9d98419182f133302b83f612 2024-11-22T18:55:32,906 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/68bc0ffc9d98419182f133302b83f612, entries=150, sequenceid=53, filesize=11.7 K 2024-11-22T18:55:32,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/03902c96c6c544109a2f6077ea555546 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/03902c96c6c544109a2f6077ea555546 2024-11-22T18:55:32,923 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/03902c96c6c544109a2f6077ea555546, entries=150, sequenceid=53, filesize=11.7 K 2024-11-22T18:55:32,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/9c71c5ce5d5f4bdbabb6d078ac2814c6 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/9c71c5ce5d5f4bdbabb6d078ac2814c6 2024-11-22T18:55:32,948 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/9c71c5ce5d5f4bdbabb6d078ac2814c6, entries=150, sequenceid=53, filesize=11.7 K 2024-11-22T18:55:32,950 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 0fc957929e5be1c291085a12bd8cf851 in 633ms, sequenceid=53, compaction requested=true 2024-11-22T18:55:32,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0fc957929e5be1c291085a12bd8cf851: 2024-11-22T18:55:32,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fc957929e5be1c291085a12bd8cf851:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T18:55:32,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:55:32,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fc957929e5be1c291085a12bd8cf851:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T18:55:32,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:55:32,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fc957929e5be1c291085a12bd8cf851:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T18:55:32,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:55:32,965 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T18:55:32,969 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T18:55:32,971 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1541): 0fc957929e5be1c291085a12bd8cf851/B is initiating minor compaction (all files) 2024-11-22T18:55:32,972 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fc957929e5be1c291085a12bd8cf851/B in TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:32,972 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/6616ca09ab9e456c9f71da3c13334460, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/5304e9a1622746cd996429ba8355d625, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/03902c96c6c544109a2f6077ea555546] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp, totalSize=35.2 K 2024-11-22T18:55:32,974 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 6616ca09ab9e456c9f71da3c13334460, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732301730550 2024-11-22T18:55:32,975 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 5304e9a1622746cd996429ba8355d625, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732301730746 2024-11-22T18:55:32,975 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 03902c96c6c544109a2f6077ea555546, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732301731966 2024-11-22T18:55:32,977 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T18:55:32,980 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T18:55:32,980 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1541): 0fc957929e5be1c291085a12bd8cf851/A is initiating minor compaction (all files) 2024-11-22T18:55:32,980 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fc957929e5be1c291085a12bd8cf851/A in TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:32,981 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/530a42fe465c4de3aa16c654d0b2d64b, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/1f4e77d17cbe4deead62ce1933cb5be1, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/68bc0ffc9d98419182f133302b83f612] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp, totalSize=35.2 K 2024-11-22T18:55:32,986 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 530a42fe465c4de3aa16c654d0b2d64b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732301730550 2024-11-22T18:55:32,987 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f4e77d17cbe4deead62ce1933cb5be1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732301730746 2024-11-22T18:55:32,988 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68bc0ffc9d98419182f133302b83f612, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732301731966 2024-11-22T18:55:33,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-22T18:55:33,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:33,007 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 0fc957929e5be1c291085a12bd8cf851 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-22T18:55:33,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-22T18:55:33,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on 0fc957929e5be1c291085a12bd8cf851 2024-11-22T18:55:33,029 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. as already flushing 2024-11-22T18:55:33,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/df4993160e9c4b96a65479bb49bfd920 is 50, key is test_row_0/A:col10/1732301732364/Put/seqid=0 2024-11-22T18:55:33,062 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fc957929e5be1c291085a12bd8cf851#A#compaction#10 average throughput is 0.33 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:55:33,063 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/170069664e754b08980f680460e8fdfb is 50, key is test_row_0/A:col10/1732301731985/Put/seqid=0 2024-11-22T18:55:33,067 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fc957929e5be1c291085a12bd8cf851#B#compaction#11 average throughput is 0.39 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:55:33,068 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/167c457aadce411797b9242c311237b1 is 50, key is test_row_0/B:col10/1732301731985/Put/seqid=0 2024-11-22T18:55:33,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741846_1022 (size=12001) 2024-11-22T18:55:33,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741847_1023 (size=12104) 2024-11-22T18:55:33,112 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/170069664e754b08980f680460e8fdfb as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/170069664e754b08980f680460e8fdfb 2024-11-22T18:55:33,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:33,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40756 deadline: 1732301793099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:33,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741848_1024 (size=12104) 2024-11-22T18:55:33,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:33,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:33,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:33,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:33,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1732301793123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:33,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:33,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:33,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:33,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:33,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40746 deadline: 1732301793123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:33,131 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:33,131 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:33,131 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:33,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:33,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40782 deadline: 1732301793120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:33,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:33,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:33,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:33,141 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/167c457aadce411797b9242c311237b1 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/167c457aadce411797b9242c311237b1 2024-11-22T18:55:33,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:33,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40758 deadline: 1732301793126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:33,148 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:33,148 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:33,149 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:33,156 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fc957929e5be1c291085a12bd8cf851/A of 0fc957929e5be1c291085a12bd8cf851 into 170069664e754b08980f680460e8fdfb(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:55:33,156 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fc957929e5be1c291085a12bd8cf851/B of 0fc957929e5be1c291085a12bd8cf851 into 167c457aadce411797b9242c311237b1(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:55:33,156 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fc957929e5be1c291085a12bd8cf851: 2024-11-22T18:55:33,156 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fc957929e5be1c291085a12bd8cf851: 2024-11-22T18:55:33,159 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., storeName=0fc957929e5be1c291085a12bd8cf851/B, priority=13, startTime=1732301732963; duration=0sec 2024-11-22T18:55:33,159 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., storeName=0fc957929e5be1c291085a12bd8cf851/A, priority=13, startTime=1732301732952; duration=0sec 2024-11-22T18:55:33,160 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:55:33,160 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:55:33,160 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fc957929e5be1c291085a12bd8cf851:B 2024-11-22T18:55:33,160 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fc957929e5be1c291085a12bd8cf851:A 2024-11-22T18:55:33,160 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T18:55:33,164 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T18:55:33,164 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1541): 0fc957929e5be1c291085a12bd8cf851/C is initiating minor compaction (all files) 2024-11-22T18:55:33,165 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fc957929e5be1c291085a12bd8cf851/C in TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:33,165 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/c45836aee3c643dfa7a9f228a0926a2b, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/ffbe2484e7754ff089bb9b8da23c538d, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/9c71c5ce5d5f4bdbabb6d078ac2814c6] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp, totalSize=35.2 K 2024-11-22T18:55:33,166 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting c45836aee3c643dfa7a9f228a0926a2b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732301730550 2024-11-22T18:55:33,167 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting ffbe2484e7754ff089bb9b8da23c538d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732301730746 2024-11-22T18:55:33,168 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c71c5ce5d5f4bdbabb6d078ac2814c6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732301731966 2024-11-22T18:55:33,196 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fc957929e5be1c291085a12bd8cf851#C#compaction#12 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:55:33,197 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/f1bd7a68a09b48cd9ad291aa308f0689 is 50, key is test_row_0/C:col10/1732301731985/Put/seqid=0 2024-11-22T18:55:33,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741849_1025 (size=12104) 2024-11-22T18:55:33,240 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/f1bd7a68a09b48cd9ad291aa308f0689 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/f1bd7a68a09b48cd9ad291aa308f0689 2024-11-22T18:55:33,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:33,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1732301793237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:33,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:33,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:33,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:33,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:33,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40756 deadline: 1732301793237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:33,245 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:33,245 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:33,245 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:33,245 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:33,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40746 deadline: 1732301793237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:33,246 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:33,246 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:33,246 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:33,256 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fc957929e5be1c291085a12bd8cf851/C of 0fc957929e5be1c291085a12bd8cf851 into f1bd7a68a09b48cd9ad291aa308f0689(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:55:33,256 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fc957929e5be1c291085a12bd8cf851: 2024-11-22T18:55:33,256 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., storeName=0fc957929e5be1c291085a12bd8cf851/C, priority=13, startTime=1732301732963; duration=0sec 2024-11-22T18:55:33,256 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:55:33,256 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fc957929e5be1c291085a12bd8cf851:C 2024-11-22T18:55:33,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:33,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40782 deadline: 1732301793246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:33,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:33,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:33,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:33,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:33,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40758 deadline: 1732301793256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:33,268 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:33,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:33,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:33,328 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-22T18:55:33,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-22T18:55:33,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:33,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1732301793456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:33,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:33,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:33,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:33,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:33,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40746 deadline: 1732301793457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:33,467 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:33,467 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:33,467 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:33,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:33,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40756 deadline: 1732301793457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:33,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:33,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:33,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:33,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:33,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40758 deadline: 1732301793476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:33,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:33,482 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:33,483 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:33,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40782 deadline: 1732301793476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:33,483 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:33,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:33,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:33,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:33,490 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/df4993160e9c4b96a65479bb49bfd920 2024-11-22T18:55:33,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/30c3bcc79b0b4d079ea0087669192a67 is 50, key is test_row_0/B:col10/1732301732364/Put/seqid=0 2024-11-22T18:55:33,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741850_1026 (size=12001) 2024-11-22T18:55:33,554 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/30c3bcc79b0b4d079ea0087669192a67 2024-11-22T18:55:33,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/1946508858304f98b3f42766981dc77b is 50, key is test_row_0/C:col10/1732301732364/Put/seqid=0 2024-11-22T18:55:33,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741851_1027 (size=12001) 2024-11-22T18:55:33,610 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/1946508858304f98b3f42766981dc77b 2024-11-22T18:55:33,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/df4993160e9c4b96a65479bb49bfd920 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/df4993160e9c4b96a65479bb49bfd920 2024-11-22T18:55:33,643 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/df4993160e9c4b96a65479bb49bfd920, entries=150, sequenceid=73, filesize=11.7 K 2024-11-22T18:55:33,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/30c3bcc79b0b4d079ea0087669192a67 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/30c3bcc79b0b4d079ea0087669192a67 2024-11-22T18:55:33,667 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/30c3bcc79b0b4d079ea0087669192a67, entries=150, sequenceid=73, filesize=11.7 K 2024-11-22T18:55:33,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/1946508858304f98b3f42766981dc77b as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/1946508858304f98b3f42766981dc77b 2024-11-22T18:55:33,689 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/1946508858304f98b3f42766981dc77b, entries=150, sequenceid=73, filesize=11.7 K 2024-11-22T18:55:33,691 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 0fc957929e5be1c291085a12bd8cf851 in 683ms, sequenceid=73, compaction requested=false 2024-11-22T18:55:33,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 0fc957929e5be1c291085a12bd8cf851: 2024-11-22T18:55:33,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:33,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-22T18:55:33,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-22T18:55:33,700 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-22T18:55:33,700 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0000 sec 2024-11-22T18:55:33,706 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.0130 sec 2024-11-22T18:55:33,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on 0fc957929e5be1c291085a12bd8cf851 2024-11-22T18:55:33,780 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T18:55:33,781 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0fc957929e5be1c291085a12bd8cf851 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-22T18:55:33,804 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/e75afde1f7d542139c31960498aeb306 is 50, key is test_row_0/A:col10/1732301733777/Put/seqid=0 2024-11-22T18:55:33,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741852_1028 (size=14341) 2024-11-22T18:55:33,826 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/e75afde1f7d542139c31960498aeb306 2024-11-22T18:55:33,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:33,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1732301793821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:33,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:33,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:33,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:33,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:33,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40756 deadline: 1732301793822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:33,832 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:33,832 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:33,832 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:33,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:33,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40782 deadline: 1732301793824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:33,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:33,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:33,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:33,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:33,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40746 deadline: 1732301793828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:33,836 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:33,836 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:33,836 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:33,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-22T18:55:33,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:33,837 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T18:55:33,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40758 deadline: 1732301793830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:33,839 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:33,839 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T18:55:33,839 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:33,839 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:33,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-22T18:55:33,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-22T18:55:33,842 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T18:55:33,845 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T18:55:33,845 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T18:55:33,849 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/7d86e1721dc24b519cae934041cb5b8f is 50, key is test_row_0/B:col10/1732301733777/Put/seqid=0 2024-11-22T18:55:33,854 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-22T18:55:33,855 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-22T18:55:33,858 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-22T18:55:33,858 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-22T18:55:33,858 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-22T18:55:33,858 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-22T18:55:33,859 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-22T18:55:33,859 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-22T18:55:33,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741853_1029 (size=12001) 2024-11-22T18:55:33,936 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:33,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:33,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40756 deadline: 1732301793936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:33,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40782 deadline: 1732301793936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:33,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:33,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:33,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1732301793936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:33,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:33,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:33,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:33,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:33,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:33,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:33,940 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:33,940 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:33,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-22T18:55:33,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:33,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40746 deadline: 1732301793946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:33,952 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:33,952 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:33,952 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:33,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:33,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40758 deadline: 1732301793946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:33,954 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:33,954 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:33,954 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:34,001 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-22T18:55:34,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:34,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. as already flushing 2024-11-22T18:55:34,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:34,002 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:55:34,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:55:34,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:55:34,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-22T18:55:34,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:34,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40756 deadline: 1732301794155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:34,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:34,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:34,159 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-22T18:55:34,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:34,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:34,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. as already flushing 2024-11-22T18:55:34,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:34,160 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:55:34,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:55:34,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:34,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1732301794155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:34,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:34,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40746 deadline: 1732301794157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:34,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:34,164 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:34,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:34,164 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:34,164 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:34,164 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:34,164 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:34,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40782 deadline: 1732301794156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:34,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:34,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:34,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:34,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:55:34,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:34,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40758 deadline: 1732301794174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:34,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:34,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:34,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:34,277 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/7d86e1721dc24b519cae934041cb5b8f 2024-11-22T18:55:34,300 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/0528648d1a5544fb8b66cdc77460ed48 is 50, key is test_row_0/C:col10/1732301733777/Put/seqid=0 2024-11-22T18:55:34,319 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-22T18:55:34,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:34,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. as already flushing 2024-11-22T18:55:34,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:34,324 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:55:34,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:55:34,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:55:34,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741854_1030 (size=12001) 2024-11-22T18:55:34,351 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/0528648d1a5544fb8b66cdc77460ed48 2024-11-22T18:55:34,364 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/e75afde1f7d542139c31960498aeb306 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/e75afde1f7d542139c31960498aeb306 2024-11-22T18:55:34,377 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/e75afde1f7d542139c31960498aeb306, entries=200, sequenceid=93, filesize=14.0 K 2024-11-22T18:55:34,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/7d86e1721dc24b519cae934041cb5b8f as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/7d86e1721dc24b519cae934041cb5b8f 2024-11-22T18:55:34,389 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/7d86e1721dc24b519cae934041cb5b8f, entries=150, sequenceid=93, filesize=11.7 K 2024-11-22T18:55:34,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/0528648d1a5544fb8b66cdc77460ed48 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/0528648d1a5544fb8b66cdc77460ed48 2024-11-22T18:55:34,401 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/0528648d1a5544fb8b66cdc77460ed48, entries=150, sequenceid=93, filesize=11.7 K 2024-11-22T18:55:34,403 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=114.05 KB/116790 for 0fc957929e5be1c291085a12bd8cf851 in 623ms, sequenceid=93, compaction requested=true 2024-11-22T18:55:34,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0fc957929e5be1c291085a12bd8cf851: 2024-11-22T18:55:34,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fc957929e5be1c291085a12bd8cf851:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T18:55:34,403 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T18:55:34,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:55:34,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fc957929e5be1c291085a12bd8cf851:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T18:55:34,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:55:34,404 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T18:55:34,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fc957929e5be1c291085a12bd8cf851:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T18:55:34,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:55:34,406 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T18:55:34,406 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1541): 0fc957929e5be1c291085a12bd8cf851/A is initiating minor compaction (all files) 2024-11-22T18:55:34,406 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T18:55:34,406 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1541): 0fc957929e5be1c291085a12bd8cf851/B is initiating minor compaction (all files) 2024-11-22T18:55:34,406 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fc957929e5be1c291085a12bd8cf851/A in TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:34,406 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fc957929e5be1c291085a12bd8cf851/B in TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:34,406 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/170069664e754b08980f680460e8fdfb, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/df4993160e9c4b96a65479bb49bfd920, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/e75afde1f7d542139c31960498aeb306] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp, totalSize=37.5 K 2024-11-22T18:55:34,406 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/167c457aadce411797b9242c311237b1, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/30c3bcc79b0b4d079ea0087669192a67, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/7d86e1721dc24b519cae934041cb5b8f] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp, totalSize=35.3 K 2024-11-22T18:55:34,407 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 170069664e754b08980f680460e8fdfb, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732301731966 2024-11-22T18:55:34,407 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 167c457aadce411797b9242c311237b1, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732301731966 2024-11-22T18:55:34,408 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting df4993160e9c4b96a65479bb49bfd920, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1732301732364 2024-11-22T18:55:34,408 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 30c3bcc79b0b4d079ea0087669192a67, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1732301732364 2024-11-22T18:55:34,408 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting e75afde1f7d542139c31960498aeb306, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732301733070 2024-11-22T18:55:34,410 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d86e1721dc24b519cae934041cb5b8f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732301733070 2024-11-22T18:55:34,435 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fc957929e5be1c291085a12bd8cf851#B#compaction#18 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:55:34,436 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/e6b9a91556e0420b8ebc8739eb3c29e2 is 50, key is test_row_0/B:col10/1732301733777/Put/seqid=0 2024-11-22T18:55:34,448 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fc957929e5be1c291085a12bd8cf851#A#compaction#19 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:55:34,449 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/c773ca82cb6d4c0d909aa064b2a0413d is 50, key is test_row_0/A:col10/1732301733777/Put/seqid=0 2024-11-22T18:55:34,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-22T18:55:34,479 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-22T18:55:34,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:34,484 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 0fc957929e5be1c291085a12bd8cf851 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-22T18:55:34,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on 0fc957929e5be1c291085a12bd8cf851 2024-11-22T18:55:34,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. as already flushing 2024-11-22T18:55:34,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741855_1031 (size=12207) 2024-11-22T18:55:34,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741856_1032 (size=12207) 2024-11-22T18:55:34,503 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/c773ca82cb6d4c0d909aa064b2a0413d as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/c773ca82cb6d4c0d909aa064b2a0413d 2024-11-22T18:55:34,512 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/e6b9a91556e0420b8ebc8739eb3c29e2 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/e6b9a91556e0420b8ebc8739eb3c29e2 2024-11-22T18:55:34,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/5b59a42053de4c5aba20580e18de719a is 50, key is test_row_0/A:col10/1732301733820/Put/seqid=0 2024-11-22T18:55:34,519 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fc957929e5be1c291085a12bd8cf851/A of 0fc957929e5be1c291085a12bd8cf851 into c773ca82cb6d4c0d909aa064b2a0413d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:55:34,519 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fc957929e5be1c291085a12bd8cf851: 2024-11-22T18:55:34,520 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., storeName=0fc957929e5be1c291085a12bd8cf851/A, priority=13, startTime=1732301734403; duration=0sec 2024-11-22T18:55:34,520 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:55:34,520 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fc957929e5be1c291085a12bd8cf851:A 2024-11-22T18:55:34,520 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T18:55:34,522 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T18:55:34,522 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1541): 0fc957929e5be1c291085a12bd8cf851/C is initiating minor compaction (all files) 2024-11-22T18:55:34,523 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fc957929e5be1c291085a12bd8cf851/C in TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:34,523 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/f1bd7a68a09b48cd9ad291aa308f0689, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/1946508858304f98b3f42766981dc77b, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/0528648d1a5544fb8b66cdc77460ed48] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp, totalSize=35.3 K 2024-11-22T18:55:34,524 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1bd7a68a09b48cd9ad291aa308f0689, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732301731966 2024-11-22T18:55:34,525 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1946508858304f98b3f42766981dc77b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1732301732364 2024-11-22T18:55:34,526 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0528648d1a5544fb8b66cdc77460ed48, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732301733070 2024-11-22T18:55:34,538 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fc957929e5be1c291085a12bd8cf851/B of 0fc957929e5be1c291085a12bd8cf851 into e6b9a91556e0420b8ebc8739eb3c29e2(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:55:34,538 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fc957929e5be1c291085a12bd8cf851: 2024-11-22T18:55:34,538 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., storeName=0fc957929e5be1c291085a12bd8cf851/B, priority=13, startTime=1732301734403; duration=0sec 2024-11-22T18:55:34,539 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:55:34,540 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fc957929e5be1c291085a12bd8cf851:B 2024-11-22T18:55:34,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:34,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40758 deadline: 1732301794526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:34,542 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:34,543 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:34,544 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:34,544 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:34,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40756 deadline: 1732301794524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:34,544 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fc957929e5be1c291085a12bd8cf851#C#compaction#21 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:55:34,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:34,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:34,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:34,545 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/71e663aa1a024fe09b66c0985ded942c is 50, key is test_row_0/C:col10/1732301733777/Put/seqid=0 2024-11-22T18:55:34,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741857_1033 (size=14341) 2024-11-22T18:55:34,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:34,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1732301794541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:34,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:34,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:34,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:34,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:34,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40782 deadline: 1732301794543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:34,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:34,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:34,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:34,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:34,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40746 deadline: 1732301794547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:34,555 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:34,555 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:34,555 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:34,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741858_1034 (size=12207) 2024-11-22T18:55:34,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:34,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40782 deadline: 1732301794657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:34,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:34,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:34,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:34,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:34,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:34,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1732301794657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:34,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40756 deadline: 1732301794657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:34,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:34,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:34,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:34,668 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:34,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:34,668 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:34,668 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:34,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40758 deadline: 1732301794657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:34,669 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:34,669 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:34,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:34,669 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:34,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40746 deadline: 1732301794666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:34,670 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:34,670 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:34,670 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:34,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:34,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1732301794877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:34,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:34,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:34,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:34,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:34,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40756 deadline: 1732301794877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:34,883 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:34,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:34,883 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:34,883 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:34,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40782 deadline: 1732301794877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:34,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:34,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:34,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:34,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:34,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40746 deadline: 1732301794878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:34,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:34,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40758 deadline: 1732301794878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:34,887 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:34,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:34,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:34,889 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:34,889 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:34,889 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:34,951 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/5b59a42053de4c5aba20580e18de719a 2024-11-22T18:55:34,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/db5902d6b174425dbc93bde571672e15 is 50, key is test_row_0/B:col10/1732301733820/Put/seqid=0 2024-11-22T18:55:34,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-22T18:55:35,016 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/71e663aa1a024fe09b66c0985ded942c as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/71e663aa1a024fe09b66c0985ded942c 2024-11-22T18:55:35,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741859_1035 (size=12001) 2024-11-22T18:55:35,039 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fc957929e5be1c291085a12bd8cf851/C of 0fc957929e5be1c291085a12bd8cf851 into 71e663aa1a024fe09b66c0985ded942c(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:55:35,039 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fc957929e5be1c291085a12bd8cf851: 2024-11-22T18:55:35,039 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., storeName=0fc957929e5be1c291085a12bd8cf851/C, priority=13, startTime=1732301734404; duration=0sec 2024-11-22T18:55:35,039 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:55:35,040 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fc957929e5be1c291085a12bd8cf851:C 2024-11-22T18:55:35,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:35,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1732301795195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:35,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:35,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40756 deadline: 1732301795196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:35,198 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:35,198 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:35,198 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:35,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:35,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:35,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:35,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:35,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40746 deadline: 1732301795196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:35,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:35,199 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:35,199 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:35,199 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:35,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:35,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40758 deadline: 1732301795196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:35,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40782 deadline: 1732301795196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:35,200 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:35,200 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:35,200 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:35,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:35,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:35,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:35,425 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/db5902d6b174425dbc93bde571672e15 2024-11-22T18:55:35,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/242ffc8934484b99800b8a0ab9c672cb is 50, key is test_row_0/C:col10/1732301733820/Put/seqid=0 2024-11-22T18:55:35,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741860_1036 (size=12001) 2024-11-22T18:55:35,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:35,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:35,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1732301795707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:35,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40756 deadline: 1732301795706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:35,709 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:35,709 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:35,709 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:35,709 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:35,709 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:35,709 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:35,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:35,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40758 deadline: 1732301795707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:35,710 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:35,710 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:35,710 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:35,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:35,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40746 deadline: 1732301795707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:35,712 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:35,712 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:35,712 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:35,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:35,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40782 deadline: 1732301795716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:35,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:35,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:35,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:35,850 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/242ffc8934484b99800b8a0ab9c672cb 2024-11-22T18:55:35,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/5b59a42053de4c5aba20580e18de719a as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/5b59a42053de4c5aba20580e18de719a 2024-11-22T18:55:35,867 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/5b59a42053de4c5aba20580e18de719a, entries=200, sequenceid=114, filesize=14.0 K 2024-11-22T18:55:35,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/db5902d6b174425dbc93bde571672e15 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/db5902d6b174425dbc93bde571672e15 2024-11-22T18:55:35,879 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/db5902d6b174425dbc93bde571672e15, entries=150, sequenceid=114, filesize=11.7 K 2024-11-22T18:55:35,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/242ffc8934484b99800b8a0ab9c672cb as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/242ffc8934484b99800b8a0ab9c672cb 2024-11-22T18:55:35,891 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/242ffc8934484b99800b8a0ab9c672cb, entries=150, sequenceid=114, filesize=11.7 K 2024-11-22T18:55:35,893 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=87.22 KB/89310 for 0fc957929e5be1c291085a12bd8cf851 in 1414ms, sequenceid=114, compaction requested=false 2024-11-22T18:55:35,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 0fc957929e5be1c291085a12bd8cf851: 2024-11-22T18:55:35,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:35,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-22T18:55:35,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-22T18:55:35,902 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-22T18:55:35,902 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0510 sec 2024-11-22T18:55:35,906 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 2.0640 sec 2024-11-22T18:55:35,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-22T18:55:35,986 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T18:55:35,987 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T18:55:35,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-22T18:55:35,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-22T18:55:35,990 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T18:55:35,991 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T18:55:35,992 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T18:55:36,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-22T18:55:36,146 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-22T18:55:36,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:36,146 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 0fc957929e5be1c291085a12bd8cf851 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-22T18:55:36,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/dbf1f38b0c1c497ba1b9976ab9c80301 is 50, key is test_row_0/A:col10/1732301734539/Put/seqid=0 2024-11-22T18:55:36,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741861_1037 (size=9757) 2024-11-22T18:55:36,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-22T18:55:36,608 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/dbf1f38b0c1c497ba1b9976ab9c80301 2024-11-22T18:55:36,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-22T18:55:36,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/cf719ea32ba44e1f8def5f11068443c0 is 50, key is test_row_0/B:col10/1732301734539/Put/seqid=0 2024-11-22T18:55:36,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741862_1038 (size=9757) 2024-11-22T18:55:36,718 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. as already flushing 2024-11-22T18:55:36,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on 0fc957929e5be1c291085a12bd8cf851 2024-11-22T18:55:36,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:36,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1732301796803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:36,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:36,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:36,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:36,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:36,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40782 deadline: 1732301796810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:36,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:36,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40746 deadline: 1732301796806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:36,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:36,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:36,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:36,815 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:36,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40756 deadline: 1732301796810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:36,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:36,815 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:36,815 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:36,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:36,816 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:36,816 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:36,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40758 deadline: 1732301796804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:36,816 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:36,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:36,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:36,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:36,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:36,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1732301796916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:36,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:36,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:36,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:36,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:36,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40782 deadline: 1732301796925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:36,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:36,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:36,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:36,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:36,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40746 deadline: 1732301796925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:36,928 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:36,928 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:36,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:36,928 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:36,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40756 deadline: 1732301796926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:36,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:36,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:36,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:36,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:36,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40758 deadline: 1732301796926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:36,929 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:36,929 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:36,929 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:37,043 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/cf719ea32ba44e1f8def5f11068443c0 2024-11-22T18:55:37,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/0973836890f045978d0d7961f119fb26 is 50, key is test_row_0/C:col10/1732301734539/Put/seqid=0 2024-11-22T18:55:37,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741863_1039 (size=9757) 2024-11-22T18:55:37,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-22T18:55:37,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:37,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1732301797126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:37,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:37,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:37,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:37,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:37,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40782 deadline: 1732301797136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:37,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:37,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:37,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:37,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40756 deadline: 1732301797137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:37,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:37,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:37,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:37,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:37,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:37,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:37,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40746 deadline: 1732301797136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:37,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40758 deadline: 1732301797137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:37,139 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:37,139 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:37,139 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:37,139 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:37,139 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:37,139 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:37,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:37,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40756 deadline: 1732301797446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:37,448 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:37,449 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:37,449 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:37,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:37,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:37,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1732301797446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:37,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40758 deadline: 1732301797446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:37,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:37,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:37,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:37,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:37,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40782 deadline: 1732301797447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:37,450 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:37,450 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:37,450 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:37,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:37,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:37,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:37,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:37,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40746 deadline: 1732301797447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:37,451 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:37,451 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:37,451 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:37,474 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/0973836890f045978d0d7961f119fb26 2024-11-22T18:55:37,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/dbf1f38b0c1c497ba1b9976ab9c80301 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/dbf1f38b0c1c497ba1b9976ab9c80301 2024-11-22T18:55:37,498 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/dbf1f38b0c1c497ba1b9976ab9c80301, entries=100, sequenceid=133, filesize=9.5 K 2024-11-22T18:55:37,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/cf719ea32ba44e1f8def5f11068443c0 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/cf719ea32ba44e1f8def5f11068443c0 2024-11-22T18:55:37,510 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/cf719ea32ba44e1f8def5f11068443c0, entries=100, sequenceid=133, filesize=9.5 K 2024-11-22T18:55:37,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/0973836890f045978d0d7961f119fb26 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/0973836890f045978d0d7961f119fb26 2024-11-22T18:55:37,523 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/0973836890f045978d0d7961f119fb26, entries=100, sequenceid=133, filesize=9.5 K 2024-11-22T18:55:37,525 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for 0fc957929e5be1c291085a12bd8cf851 in 1379ms, sequenceid=133, compaction requested=true 2024-11-22T18:55:37,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 0fc957929e5be1c291085a12bd8cf851: 2024-11-22T18:55:37,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:37,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-22T18:55:37,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-22T18:55:37,532 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-22T18:55:37,532 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5370 sec 2024-11-22T18:55:37,537 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.5460 sec 2024-11-22T18:55:37,966 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T18:55:37,966 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0fc957929e5be1c291085a12bd8cf851 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-22T18:55:37,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on 0fc957929e5be1c291085a12bd8cf851 2024-11-22T18:55:37,986 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/76f5cc1dc97f41779e3ce0d0c0336ca3 is 50, key is test_row_0/A:col10/1732301737965/Put/seqid=0 2024-11-22T18:55:38,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741864_1040 (size=12151) 2024-11-22T18:55:38,031 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/76f5cc1dc97f41779e3ce0d0c0336ca3 2024-11-22T18:55:38,046 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/7662f722c1874f70a8d83539e7029011 is 50, key is test_row_0/B:col10/1732301737965/Put/seqid=0 2024-11-22T18:55:38,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741865_1041 (size=12151) 2024-11-22T18:55:38,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:38,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:38,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40756 deadline: 1732301798003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:38,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40746 deadline: 1732301797995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:38,076 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:38,076 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:38,076 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:38,077 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:38,077 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:38,077 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:38,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:38,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40782 deadline: 1732301798071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:38,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:38,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:38,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:38,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:38,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1732301798072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:38,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:38,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:38,085 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:38,085 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:38,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40758 deadline: 1732301798077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:38,085 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:38,085 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:38,086 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:38,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-22T18:55:38,138 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T18:55:38,138 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T18:55:38,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-22T18:55:38,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-22T18:55:38,144 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T18:55:38,145 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T18:55:38,145 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T18:55:38,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:38,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40782 deadline: 1732301798186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:38,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:38,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40746 deadline: 1732301798186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:38,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:38,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:38,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:38,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:38,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40756 deadline: 1732301798186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:38,190 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:38,190 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:38,190 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:38,190 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:38,190 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:38,190 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:38,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:38,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:38,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1732301798196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:38,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40758 deadline: 1732301798196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:38,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:38,201 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:38,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:38,201 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:38,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:38,201 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:38,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-22T18:55:38,308 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-22T18:55:38,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:38,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. as already flushing 2024-11-22T18:55:38,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:38,309 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:55:38,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:55:38,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:55:38,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:38,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40756 deadline: 1732301798396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:38,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:38,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:38,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:38,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40782 deadline: 1732301798396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:38,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:38,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:38,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:38,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40746 deadline: 1732301798396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:38,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:38,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:38,402 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:38,402 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:38,402 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:38,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:38,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40758 deadline: 1732301798405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:38,415 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:38,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:38,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:38,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:38,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1732301798405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:38,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:38,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:38,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:38,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-22T18:55:38,463 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-22T18:55:38,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:38,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. as already flushing 2024-11-22T18:55:38,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:38,464 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:55:38,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:55:38,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:55:38,468 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/7662f722c1874f70a8d83539e7029011 2024-11-22T18:55:38,496 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/e7de47761f2846ed850d07bdb274fea6 is 50, key is test_row_0/C:col10/1732301737965/Put/seqid=0 2024-11-22T18:55:38,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741866_1042 (size=12151) 2024-11-22T18:55:38,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-22T18:55:38,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:38,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. as already flushing 2024-11-22T18:55:38,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:38,619 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:55:38,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:55:38,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:55:38,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:38,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40756 deadline: 1732301798705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:38,710 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:38,710 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:38,710 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:38,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:38,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40782 deadline: 1732301798707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:38,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:38,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:38,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:38,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:38,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40746 deadline: 1732301798716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:38,721 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:38,721 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:38,721 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:38,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:38,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1732301798727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:38,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:38,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40758 deadline: 1732301798727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:38,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:38,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:38,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:38,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:38,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:38,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:38,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-22T18:55:38,776 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-22T18:55:38,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:38,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. as already flushing 2024-11-22T18:55:38,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:38,776 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:55:38,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:55:38,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:55:38,906 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/e7de47761f2846ed850d07bdb274fea6 2024-11-22T18:55:38,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/76f5cc1dc97f41779e3ce0d0c0336ca3 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/76f5cc1dc97f41779e3ce0d0c0336ca3 2024-11-22T18:55:38,931 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-22T18:55:38,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:38,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. as already flushing 2024-11-22T18:55:38,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:38,932 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:55:38,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:55:38,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:55:38,940 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/76f5cc1dc97f41779e3ce0d0c0336ca3, entries=150, sequenceid=157, filesize=11.9 K 2024-11-22T18:55:38,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/7662f722c1874f70a8d83539e7029011 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/7662f722c1874f70a8d83539e7029011 2024-11-22T18:55:38,953 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/7662f722c1874f70a8d83539e7029011, entries=150, sequenceid=157, filesize=11.9 K 2024-11-22T18:55:38,955 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/e7de47761f2846ed850d07bdb274fea6 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/e7de47761f2846ed850d07bdb274fea6 2024-11-22T18:55:38,965 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/e7de47761f2846ed850d07bdb274fea6, entries=150, sequenceid=157, filesize=11.9 K 2024-11-22T18:55:38,967 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 0fc957929e5be1c291085a12bd8cf851 in 1000ms, sequenceid=157, compaction requested=true 2024-11-22T18:55:38,967 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0fc957929e5be1c291085a12bd8cf851: 2024-11-22T18:55:38,967 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T18:55:38,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fc957929e5be1c291085a12bd8cf851:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T18:55:38,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:55:38,968 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T18:55:38,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fc957929e5be1c291085a12bd8cf851:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T18:55:38,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:55:38,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fc957929e5be1c291085a12bd8cf851:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T18:55:38,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:55:38,974 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46116 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T18:55:38,974 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1541): 0fc957929e5be1c291085a12bd8cf851/B is initiating minor compaction (all files) 2024-11-22T18:55:38,974 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fc957929e5be1c291085a12bd8cf851/B in TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:38,974 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/e6b9a91556e0420b8ebc8739eb3c29e2, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/db5902d6b174425dbc93bde571672e15, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/cf719ea32ba44e1f8def5f11068443c0, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/7662f722c1874f70a8d83539e7029011] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp, totalSize=45.0 K 2024-11-22T18:55:38,975 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting e6b9a91556e0420b8ebc8739eb3c29e2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732301733070 2024-11-22T18:55:38,975 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48456 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T18:55:38,975 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting db5902d6b174425dbc93bde571672e15, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732301733820 2024-11-22T18:55:38,976 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1541): 0fc957929e5be1c291085a12bd8cf851/A is initiating minor compaction (all files) 2024-11-22T18:55:38,976 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fc957929e5be1c291085a12bd8cf851/A in TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:38,976 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/c773ca82cb6d4c0d909aa064b2a0413d, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/5b59a42053de4c5aba20580e18de719a, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/dbf1f38b0c1c497ba1b9976ab9c80301, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/76f5cc1dc97f41779e3ce0d0c0336ca3] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp, totalSize=47.3 K 2024-11-22T18:55:38,976 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting cf719ea32ba44e1f8def5f11068443c0, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732301734539 2024-11-22T18:55:38,976 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting c773ca82cb6d4c0d909aa064b2a0413d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732301733070 2024-11-22T18:55:38,977 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 7662f722c1874f70a8d83539e7029011, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732301736792 2024-11-22T18:55:38,977 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b59a42053de4c5aba20580e18de719a, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732301733817 2024-11-22T18:55:38,978 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting dbf1f38b0c1c497ba1b9976ab9c80301, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732301734539 2024-11-22T18:55:38,981 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76f5cc1dc97f41779e3ce0d0c0336ca3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732301736792 2024-11-22T18:55:39,022 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fc957929e5be1c291085a12bd8cf851#B#compaction#30 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:55:39,023 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/a18bc7005e9c435ea64372bb15738d7d is 50, key is test_row_0/B:col10/1732301737965/Put/seqid=0 2024-11-22T18:55:39,031 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fc957929e5be1c291085a12bd8cf851#A#compaction#31 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:55:39,032 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/f1a3ac8e9d464b5a906d51bd26a808c8 is 50, key is test_row_0/A:col10/1732301737965/Put/seqid=0 2024-11-22T18:55:39,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741867_1043 (size=12493) 2024-11-22T18:55:39,087 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-22T18:55:39,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:39,088 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 0fc957929e5be1c291085a12bd8cf851 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-22T18:55:39,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741868_1044 (size=12493) 2024-11-22T18:55:39,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/6d7ea4d2e3f74246b4178825a7a7e23f is 50, key is test_row_0/A:col10/1732301738000/Put/seqid=0 2024-11-22T18:55:39,124 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/f1a3ac8e9d464b5a906d51bd26a808c8 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/f1a3ac8e9d464b5a906d51bd26a808c8 2024-11-22T18:55:39,137 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0fc957929e5be1c291085a12bd8cf851/A of 0fc957929e5be1c291085a12bd8cf851 into f1a3ac8e9d464b5a906d51bd26a808c8(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:55:39,137 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fc957929e5be1c291085a12bd8cf851: 2024-11-22T18:55:39,137 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., storeName=0fc957929e5be1c291085a12bd8cf851/A, priority=12, startTime=1732301738967; duration=0sec 2024-11-22T18:55:39,137 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:55:39,138 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fc957929e5be1c291085a12bd8cf851:A 2024-11-22T18:55:39,138 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T18:55:39,140 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46116 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T18:55:39,140 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1541): 0fc957929e5be1c291085a12bd8cf851/C is initiating minor compaction (all files) 2024-11-22T18:55:39,140 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fc957929e5be1c291085a12bd8cf851/C in TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:39,141 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/71e663aa1a024fe09b66c0985ded942c, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/242ffc8934484b99800b8a0ab9c672cb, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/0973836890f045978d0d7961f119fb26, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/e7de47761f2846ed850d07bdb274fea6] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp, totalSize=45.0 K 2024-11-22T18:55:39,141 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71e663aa1a024fe09b66c0985ded942c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732301733070 2024-11-22T18:55:39,142 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 242ffc8934484b99800b8a0ab9c672cb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732301733820 2024-11-22T18:55:39,142 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0973836890f045978d0d7961f119fb26, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732301734539 2024-11-22T18:55:39,144 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7de47761f2846ed850d07bdb274fea6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732301736792 2024-11-22T18:55:39,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741869_1045 (size=12151) 2024-11-22T18:55:39,182 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fc957929e5be1c291085a12bd8cf851#C#compaction#33 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:55:39,183 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/c3907bd32f484139a2611b1de63714e4 is 50, key is test_row_0/C:col10/1732301737965/Put/seqid=0 2024-11-22T18:55:39,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741870_1046 (size=12493) 2024-11-22T18:55:39,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on 0fc957929e5be1c291085a12bd8cf851 2024-11-22T18:55:39,225 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. as already flushing 2024-11-22T18:55:39,232 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/c3907bd32f484139a2611b1de63714e4 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/c3907bd32f484139a2611b1de63714e4 2024-11-22T18:55:39,242 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0fc957929e5be1c291085a12bd8cf851/C of 0fc957929e5be1c291085a12bd8cf851 into c3907bd32f484139a2611b1de63714e4(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:55:39,243 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fc957929e5be1c291085a12bd8cf851: 2024-11-22T18:55:39,243 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., storeName=0fc957929e5be1c291085a12bd8cf851/C, priority=12, startTime=1732301738969; duration=0sec 2024-11-22T18:55:39,243 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:55:39,243 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fc957929e5be1c291085a12bd8cf851:C 2024-11-22T18:55:39,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-22T18:55:39,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:39,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40756 deadline: 1732301799292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:39,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:39,307 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:39,307 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:39,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40758 deadline: 1732301799292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:39,307 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:39,308 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:39,308 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:39,308 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:39,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:39,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1732301799300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:39,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:39,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:39,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:39,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:39,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:39,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40746 deadline: 1732301799306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:39,318 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:39,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40782 deadline: 1732301799297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:39,318 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:39,318 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:39,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:39,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:39,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:39,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:39,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40756 deadline: 1732301799416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:39,417 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:39,417 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:39,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:39,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:39,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40758 deadline: 1732301799416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:39,418 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:39,419 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:39,419 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:39,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:39,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1732301799426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:39,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:39,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:39,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:39,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:39,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40746 deadline: 1732301799426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:39,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:39,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40782 deadline: 1732301799426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:39,429 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:39,429 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:39,429 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:39,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:39,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:39,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:39,489 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/a18bc7005e9c435ea64372bb15738d7d as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/a18bc7005e9c435ea64372bb15738d7d 2024-11-22T18:55:39,502 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0fc957929e5be1c291085a12bd8cf851/B of 0fc957929e5be1c291085a12bd8cf851 into a18bc7005e9c435ea64372bb15738d7d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:55:39,502 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fc957929e5be1c291085a12bd8cf851: 2024-11-22T18:55:39,502 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., storeName=0fc957929e5be1c291085a12bd8cf851/B, priority=12, startTime=1732301738968; duration=0sec 2024-11-22T18:55:39,503 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:55:39,503 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fc957929e5be1c291085a12bd8cf851:B 2024-11-22T18:55:39,585 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/6d7ea4d2e3f74246b4178825a7a7e23f 2024-11-22T18:55:39,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/39730ab81f104679bb8bbf6f56b15c60 is 50, key is test_row_0/B:col10/1732301738000/Put/seqid=0 2024-11-22T18:55:39,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741871_1047 (size=12151) 2024-11-22T18:55:39,621 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/39730ab81f104679bb8bbf6f56b15c60 2024-11-22T18:55:39,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:39,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40758 deadline: 1732301799626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:39,627 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:39,627 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:39,627 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:39,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:39,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40756 deadline: 1732301799626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:39,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:39,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:39,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:39,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/dd50a222a7834a05bcb2e20808f050c2 is 50, key is test_row_0/C:col10/1732301738000/Put/seqid=0 2024-11-22T18:55:39,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:39,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40782 deadline: 1732301799636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:39,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:39,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:39,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:39,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:39,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1732301799636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:39,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:39,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:39,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:39,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:39,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40746 deadline: 1732301799637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:39,645 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:39,645 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:39,645 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:39,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741872_1048 (size=12151) 2024-11-22T18:55:39,662 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/dd50a222a7834a05bcb2e20808f050c2 2024-11-22T18:55:39,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/6d7ea4d2e3f74246b4178825a7a7e23f as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/6d7ea4d2e3f74246b4178825a7a7e23f 2024-11-22T18:55:39,692 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/6d7ea4d2e3f74246b4178825a7a7e23f, entries=150, sequenceid=169, filesize=11.9 K 2024-11-22T18:55:39,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/39730ab81f104679bb8bbf6f56b15c60 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/39730ab81f104679bb8bbf6f56b15c60 2024-11-22T18:55:39,702 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/39730ab81f104679bb8bbf6f56b15c60, entries=150, sequenceid=169, filesize=11.9 K 2024-11-22T18:55:39,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/dd50a222a7834a05bcb2e20808f050c2 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/dd50a222a7834a05bcb2e20808f050c2 2024-11-22T18:55:39,710 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/dd50a222a7834a05bcb2e20808f050c2, entries=150, sequenceid=169, filesize=11.9 K 2024-11-22T18:55:39,711 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 0fc957929e5be1c291085a12bd8cf851 in 623ms, sequenceid=169, compaction requested=false 2024-11-22T18:55:39,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 0fc957929e5be1c291085a12bd8cf851: 2024-11-22T18:55:39,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:39,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-22T18:55:39,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-22T18:55:39,718 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-22T18:55:39,718 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5690 sec 2024-11-22T18:55:39,721 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.5800 sec 2024-11-22T18:55:39,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on 0fc957929e5be1c291085a12bd8cf851 2024-11-22T18:55:39,939 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T18:55:39,939 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0fc957929e5be1c291085a12bd8cf851 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-22T18:55:39,952 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/3c00670513f348198d438ff1e13a25d3 is 50, key is test_row_0/A:col10/1732301739939/Put/seqid=0 2024-11-22T18:55:39,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:39,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1732301799951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:39,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:39,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:39,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:39,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:39,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40782 deadline: 1732301799955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:39,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:39,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:39,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:39,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40756 deadline: 1732301799955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:39,959 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:39,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:39,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:39,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40746 deadline: 1732301799956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:39,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:39,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:39,959 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:39,959 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:39,959 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:39,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:39,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40758 deadline: 1732301799957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:39,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:39,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:39,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:39,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741873_1049 (size=14541) 2024-11-22T18:55:39,967 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/3c00670513f348198d438ff1e13a25d3 2024-11-22T18:55:39,980 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/8c284efb46ad4e2eba4a3f1b43aa434f is 50, key is test_row_0/B:col10/1732301739939/Put/seqid=0 2024-11-22T18:55:39,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741874_1050 (size=12151) 2024-11-22T18:55:40,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:40,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40756 deadline: 1732301800066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:40,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:40,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:40,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:40,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:40,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1732301800066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:40,073 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:40,073 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:40,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:40,073 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:40,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40758 deadline: 1732301800066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:40,074 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:40,074 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:40,074 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:40,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:40,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40756 deadline: 1732301800276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:40,279 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:40,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1732301800276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:40,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:40,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:40,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:40,280 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:40,280 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:40,281 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:40,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:40,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40758 deadline: 1732301800286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:40,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:40,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:40,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:40,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-22T18:55:40,297 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T18:55:40,297 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T18:55:40,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-22T18:55:40,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-22T18:55:40,301 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T18:55:40,302 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T18:55:40,303 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T18:55:40,391 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/8c284efb46ad4e2eba4a3f1b43aa434f 2024-11-22T18:55:40,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-22T18:55:40,410 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/bc85ad73421f42d4b07a6e81ab9c4fce is 50, key is test_row_0/C:col10/1732301739939/Put/seqid=0 2024-11-22T18:55:40,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741875_1051 (size=12151) 2024-11-22T18:55:40,458 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-22T18:55:40,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:40,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. as already flushing 2024-11-22T18:55:40,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:40,458 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:55:40,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:55:40,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:55:40,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:40,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40782 deadline: 1732301800466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:40,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:40,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:40,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:40,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:40,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40746 deadline: 1732301800466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:40,470 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:40,470 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:40,470 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:40,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:40,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40756 deadline: 1732301800586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:40,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:40,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:40,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:40,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:40,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1732301800586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:40,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:40,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:40,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:40,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:40,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40758 deadline: 1732301800596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:40,598 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:40,598 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:40,598 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:40,612 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-22T18:55:40,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:40,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. as already flushing 2024-11-22T18:55:40,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:40,613 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:55:40,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:55:40,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:55:40,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-22T18:55:40,766 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-22T18:55:40,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:40,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. as already flushing 2024-11-22T18:55:40,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:40,767 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:55:40,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:55:40,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:55:40,829 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/bc85ad73421f42d4b07a6e81ab9c4fce 2024-11-22T18:55:40,837 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/3c00670513f348198d438ff1e13a25d3 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/3c00670513f348198d438ff1e13a25d3 2024-11-22T18:55:40,845 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/3c00670513f348198d438ff1e13a25d3, entries=200, sequenceid=199, filesize=14.2 K 2024-11-22T18:55:40,846 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/8c284efb46ad4e2eba4a3f1b43aa434f as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/8c284efb46ad4e2eba4a3f1b43aa434f 2024-11-22T18:55:40,865 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/8c284efb46ad4e2eba4a3f1b43aa434f, entries=150, sequenceid=199, filesize=11.9 K 2024-11-22T18:55:40,867 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/bc85ad73421f42d4b07a6e81ab9c4fce as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/bc85ad73421f42d4b07a6e81ab9c4fce 2024-11-22T18:55:40,878 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/bc85ad73421f42d4b07a6e81ab9c4fce, entries=150, sequenceid=199, filesize=11.9 K 2024-11-22T18:55:40,880 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=46.96 KB/48090 for 0fc957929e5be1c291085a12bd8cf851 in 941ms, sequenceid=199, compaction requested=true 2024-11-22T18:55:40,880 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0fc957929e5be1c291085a12bd8cf851: 2024-11-22T18:55:40,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fc957929e5be1c291085a12bd8cf851:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T18:55:40,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:55:40,880 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T18:55:40,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fc957929e5be1c291085a12bd8cf851:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T18:55:40,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:55:40,880 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T18:55:40,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fc957929e5be1c291085a12bd8cf851:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T18:55:40,882 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T18:55:40,882 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1541): 0fc957929e5be1c291085a12bd8cf851/A is initiating minor compaction (all files) 2024-11-22T18:55:40,882 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fc957929e5be1c291085a12bd8cf851/A in TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:40,882 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T18:55:40,882 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/f1a3ac8e9d464b5a906d51bd26a808c8, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/6d7ea4d2e3f74246b4178825a7a7e23f, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/3c00670513f348198d438ff1e13a25d3] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp, totalSize=38.3 K 2024-11-22T18:55:40,882 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1541): 0fc957929e5be1c291085a12bd8cf851/B is initiating minor compaction (all files) 2024-11-22T18:55:40,882 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fc957929e5be1c291085a12bd8cf851/B in TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:40,883 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/a18bc7005e9c435ea64372bb15738d7d, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/39730ab81f104679bb8bbf6f56b15c60, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/8c284efb46ad4e2eba4a3f1b43aa434f] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp, totalSize=35.9 K 2024-11-22T18:55:40,883 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1a3ac8e9d464b5a906d51bd26a808c8, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732301736792 2024-11-22T18:55:40,883 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting a18bc7005e9c435ea64372bb15738d7d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732301736792 2024-11-22T18:55:40,883 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d7ea4d2e3f74246b4178825a7a7e23f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732301738000 2024-11-22T18:55:40,884 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c00670513f348198d438ff1e13a25d3, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732301739293 2024-11-22T18:55:40,884 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 39730ab81f104679bb8bbf6f56b15c60, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732301738000 2024-11-22T18:55:40,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:55:40,885 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c284efb46ad4e2eba4a3f1b43aa434f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732301739293 2024-11-22T18:55:40,911 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fc957929e5be1c291085a12bd8cf851#A#compaction#39 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:55:40,912 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/54773f3c769643429b718ecc29ef2e35 is 50, key is test_row_0/A:col10/1732301739939/Put/seqid=0 2024-11-22T18:55:40,921 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-22T18:55:40,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:40,922 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 0fc957929e5be1c291085a12bd8cf851 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-22T18:55:40,923 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fc957929e5be1c291085a12bd8cf851#B#compaction#40 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:55:40,923 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/4cbf13b151bc4de9ae91481fe671a666 is 50, key is test_row_0/B:col10/1732301739939/Put/seqid=0 2024-11-22T18:55:40,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-22T18:55:40,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/db7d5f72fa7e4227a89a1fadc8a5b21d is 50, key is test_row_0/A:col10/1732301739949/Put/seqid=0 2024-11-22T18:55:40,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741876_1052 (size=12595) 2024-11-22T18:55:40,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741877_1053 (size=12595) 2024-11-22T18:55:40,971 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/54773f3c769643429b718ecc29ef2e35 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/54773f3c769643429b718ecc29ef2e35 2024-11-22T18:55:40,985 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/4cbf13b151bc4de9ae91481fe671a666 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/4cbf13b151bc4de9ae91481fe671a666 2024-11-22T18:55:40,986 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fc957929e5be1c291085a12bd8cf851/A of 0fc957929e5be1c291085a12bd8cf851 into 54773f3c769643429b718ecc29ef2e35(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:55:40,986 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fc957929e5be1c291085a12bd8cf851: 2024-11-22T18:55:40,986 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., storeName=0fc957929e5be1c291085a12bd8cf851/A, priority=13, startTime=1732301740880; duration=0sec 2024-11-22T18:55:40,986 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:55:40,986 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fc957929e5be1c291085a12bd8cf851:A 2024-11-22T18:55:40,986 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T18:55:40,988 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T18:55:40,988 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1541): 0fc957929e5be1c291085a12bd8cf851/C is initiating minor compaction (all files) 2024-11-22T18:55:40,988 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fc957929e5be1c291085a12bd8cf851/C in TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:40,988 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/c3907bd32f484139a2611b1de63714e4, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/dd50a222a7834a05bcb2e20808f050c2, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/bc85ad73421f42d4b07a6e81ab9c4fce] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp, totalSize=35.9 K 2024-11-22T18:55:40,989 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3907bd32f484139a2611b1de63714e4, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732301736792 2024-11-22T18:55:40,991 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd50a222a7834a05bcb2e20808f050c2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732301738000 2024-11-22T18:55:40,991 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc85ad73421f42d4b07a6e81ab9c4fce, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732301739293 2024-11-22T18:55:40,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741878_1054 (size=12151) 2024-11-22T18:55:40,996 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/db7d5f72fa7e4227a89a1fadc8a5b21d 2024-11-22T18:55:40,997 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fc957929e5be1c291085a12bd8cf851/B of 0fc957929e5be1c291085a12bd8cf851 into 4cbf13b151bc4de9ae91481fe671a666(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:55:40,997 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fc957929e5be1c291085a12bd8cf851: 2024-11-22T18:55:40,998 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., storeName=0fc957929e5be1c291085a12bd8cf851/B, priority=13, startTime=1732301740880; duration=0sec 2024-11-22T18:55:40,998 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:55:40,998 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fc957929e5be1c291085a12bd8cf851:B 2024-11-22T18:55:41,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/58c3a4207100450ead6c81637717c59b is 50, key is test_row_0/B:col10/1732301739949/Put/seqid=0 2024-11-22T18:55:41,026 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fc957929e5be1c291085a12bd8cf851#C#compaction#43 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:55:41,027 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/7d52149821a74922ae542acd73064b43 is 50, key is test_row_0/C:col10/1732301739939/Put/seqid=0 2024-11-22T18:55:41,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741879_1055 (size=12151) 2024-11-22T18:55:41,034 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/58c3a4207100450ead6c81637717c59b 2024-11-22T18:55:41,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/ca5be2fb67274703a27290081a2748ff is 50, key is test_row_0/C:col10/1732301739949/Put/seqid=0 2024-11-22T18:55:41,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741880_1056 (size=12595) 2024-11-22T18:55:41,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on 0fc957929e5be1c291085a12bd8cf851 2024-11-22T18:55:41,098 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. as already flushing 2024-11-22T18:55:41,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741881_1057 (size=12151) 2024-11-22T18:55:41,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:41,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40756 deadline: 1732301801172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:41,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:41,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:41,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:41,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:41,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40758 deadline: 1732301801174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:41,180 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:41,180 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:41,180 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:41,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:41,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1732301801180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:41,186 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:41,186 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:41,186 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:41,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:41,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40756 deadline: 1732301801285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:41,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:41,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40758 deadline: 1732301801285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:41,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:41,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:41,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:41,290 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:41,290 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:41,290 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:41,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:41,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1732301801296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:41,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:41,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:41,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:41,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-22T18:55:41,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:41,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40782 deadline: 1732301801476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:41,483 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to d79ba0c344fb,43593,1732301724173 for 'test_row_2' in 0fc957929e5be1c291085a12bd8cf851 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] ... 32 more 2024-11-22T18:55:41,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:41,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:41,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:41,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:41,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40746 deadline: 1732301801487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:41,492 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to d79ba0c344fb,43593,1732301724173 for 'test_row_2' in 0fc957929e5be1c291085a12bd8cf851 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] ... 32 more 2024-11-22T18:55:41,493 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:41,493 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:41,493 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:41,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:41,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40756 deadline: 1732301801496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:41,498 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:41,498 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:41,498 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:41,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:41,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40758 deadline: 1732301801499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:41,501 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/7d52149821a74922ae542acd73064b43 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/7d52149821a74922ae542acd73064b43 2024-11-22T18:55:41,501 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:41,501 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:41,501 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:41,503 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/ca5be2fb67274703a27290081a2748ff 2024-11-22T18:55:41,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:41,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1732301801505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:41,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:41,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:41,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:41,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/db7d5f72fa7e4227a89a1fadc8a5b21d as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/db7d5f72fa7e4227a89a1fadc8a5b21d 2024-11-22T18:55:41,514 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fc957929e5be1c291085a12bd8cf851/C of 0fc957929e5be1c291085a12bd8cf851 into 7d52149821a74922ae542acd73064b43(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:55:41,514 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fc957929e5be1c291085a12bd8cf851: 2024-11-22T18:55:41,514 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., storeName=0fc957929e5be1c291085a12bd8cf851/C, priority=13, startTime=1732301740881; duration=0sec 2024-11-22T18:55:41,515 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:55:41,515 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fc957929e5be1c291085a12bd8cf851:C 2024-11-22T18:55:41,522 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/db7d5f72fa7e4227a89a1fadc8a5b21d, entries=150, sequenceid=209, filesize=11.9 K 2024-11-22T18:55:41,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/58c3a4207100450ead6c81637717c59b as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/58c3a4207100450ead6c81637717c59b 2024-11-22T18:55:41,531 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/58c3a4207100450ead6c81637717c59b, entries=150, sequenceid=209, filesize=11.9 K 2024-11-22T18:55:41,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/ca5be2fb67274703a27290081a2748ff as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/ca5be2fb67274703a27290081a2748ff 2024-11-22T18:55:41,539 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/ca5be2fb67274703a27290081a2748ff, entries=150, sequenceid=209, filesize=11.9 K 2024-11-22T18:55:41,540 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 0fc957929e5be1c291085a12bd8cf851 in 619ms, sequenceid=209, compaction requested=false 2024-11-22T18:55:41,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 0fc957929e5be1c291085a12bd8cf851: 2024-11-22T18:55:41,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:41,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-22T18:55:41,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-22T18:55:41,548 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-22T18:55:41,548 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2410 sec 2024-11-22T18:55:41,551 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.2520 sec 2024-11-22T18:55:41,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on 0fc957929e5be1c291085a12bd8cf851 2024-11-22T18:55:41,809 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T18:55:41,809 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0fc957929e5be1c291085a12bd8cf851 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-22T18:55:41,819 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/be9abcf3a5954079bc066fd563b998a9 is 50, key is test_row_0/A:col10/1732301741176/Put/seqid=0 2024-11-22T18:55:41,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:41,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1732301801821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:41,825 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:41,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:41,825 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:41,825 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:41,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40758 deadline: 1732301801823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:41,825 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:41,825 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:41,825 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:41,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:41,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40756 deadline: 1732301801824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:41,827 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:41,827 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:41,827 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:41,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741882_1058 (size=16931) 2024-11-22T18:55:41,936 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:41,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1732301801936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:41,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:41,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:41,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:41,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:41,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40758 deadline: 1732301801936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:41,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:41,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:41,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:41,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:41,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40756 deadline: 1732301801936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:41,941 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:41,941 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:41,941 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:42,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:42,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40758 deadline: 1732301802146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:42,148 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:42,148 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:42,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:42,148 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:42,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40756 deadline: 1732301802146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:42,149 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:42,149 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:42,149 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:42,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:42,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1732301802146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:42,150 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:42,150 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:42,150 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:42,229 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/be9abcf3a5954079bc066fd563b998a9 2024-11-22T18:55:42,254 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/f5a6861a41544d3a9132cd5bcf8ee301 is 50, key is test_row_0/B:col10/1732301741176/Put/seqid=0 2024-11-22T18:55:42,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741883_1059 (size=12151) 2024-11-22T18:55:42,295 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/f5a6861a41544d3a9132cd5bcf8ee301 2024-11-22T18:55:42,313 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/611d036604e2408ba125a29638a09507 is 50, key is test_row_0/C:col10/1732301741176/Put/seqid=0 2024-11-22T18:55:42,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741884_1060 (size=12151) 2024-11-22T18:55:42,368 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/611d036604e2408ba125a29638a09507 2024-11-22T18:55:42,383 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/be9abcf3a5954079bc066fd563b998a9 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/be9abcf3a5954079bc066fd563b998a9 2024-11-22T18:55:42,391 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/be9abcf3a5954079bc066fd563b998a9, entries=250, sequenceid=240, filesize=16.5 K 2024-11-22T18:55:42,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/f5a6861a41544d3a9132cd5bcf8ee301 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/f5a6861a41544d3a9132cd5bcf8ee301 2024-11-22T18:55:42,399 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/f5a6861a41544d3a9132cd5bcf8ee301, entries=150, sequenceid=240, filesize=11.9 K 2024-11-22T18:55:42,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/611d036604e2408ba125a29638a09507 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/611d036604e2408ba125a29638a09507 2024-11-22T18:55:42,409 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/611d036604e2408ba125a29638a09507, entries=150, sequenceid=240, filesize=11.9 K 2024-11-22T18:55:42,410 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 0fc957929e5be1c291085a12bd8cf851 in 601ms, sequenceid=240, compaction requested=true 2024-11-22T18:55:42,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0fc957929e5be1c291085a12bd8cf851: 2024-11-22T18:55:42,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fc957929e5be1c291085a12bd8cf851:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T18:55:42,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:55:42,411 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T18:55:42,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fc957929e5be1c291085a12bd8cf851:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T18:55:42,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:55:42,411 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T18:55:42,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fc957929e5be1c291085a12bd8cf851:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T18:55:42,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:55:42,412 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T18:55:42,412 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41677 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T18:55:42,413 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1541): 0fc957929e5be1c291085a12bd8cf851/B is initiating minor compaction (all files) 2024-11-22T18:55:42,413 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1541): 0fc957929e5be1c291085a12bd8cf851/A is initiating minor compaction (all files) 2024-11-22T18:55:42,413 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fc957929e5be1c291085a12bd8cf851/B in TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:42,413 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fc957929e5be1c291085a12bd8cf851/A in TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:42,413 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/4cbf13b151bc4de9ae91481fe671a666, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/58c3a4207100450ead6c81637717c59b, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/f5a6861a41544d3a9132cd5bcf8ee301] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp, totalSize=36.0 K 2024-11-22T18:55:42,413 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/54773f3c769643429b718ecc29ef2e35, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/db7d5f72fa7e4227a89a1fadc8a5b21d, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/be9abcf3a5954079bc066fd563b998a9] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp, totalSize=40.7 K 2024-11-22T18:55:42,414 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54773f3c769643429b718ecc29ef2e35, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732301739293 2024-11-22T18:55:42,414 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 4cbf13b151bc4de9ae91481fe671a666, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732301739293 2024-11-22T18:55:42,414 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 58c3a4207100450ead6c81637717c59b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1732301739943 2024-11-22T18:55:42,414 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting db7d5f72fa7e4227a89a1fadc8a5b21d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1732301739943 2024-11-22T18:55:42,415 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting f5a6861a41544d3a9132cd5bcf8ee301, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1732301741176 2024-11-22T18:55:42,415 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting be9abcf3a5954079bc066fd563b998a9, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1732301741169 2024-11-22T18:55:42,439 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fc957929e5be1c291085a12bd8cf851#B#compaction#48 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:55:42,440 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/8c0885899fed4e0e95da32956ebe935c is 50, key is test_row_0/B:col10/1732301741176/Put/seqid=0 2024-11-22T18:55:42,444 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fc957929e5be1c291085a12bd8cf851#A#compaction#49 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:55:42,444 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/3f3b4f29b00f471ab26d9d33aa8c6813 is 50, key is test_row_0/A:col10/1732301741176/Put/seqid=0 2024-11-22T18:55:42,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-22T18:55:42,446 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T18:55:42,447 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T18:55:42,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-22T18:55:42,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-22T18:55:42,453 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T18:55:42,454 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T18:55:42,454 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T18:55:42,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on 0fc957929e5be1c291085a12bd8cf851 2024-11-22T18:55:42,466 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T18:55:42,466 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0fc957929e5be1c291085a12bd8cf851 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-22T18:55:42,518 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/8e27507c5e0041d8afc6ca534cc7a150 is 50, key is test_row_0/A:col10/1732301742464/Put/seqid=0 2024-11-22T18:55:42,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741885_1061 (size=12697) 2024-11-22T18:55:42,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741886_1062 (size=12697) 2024-11-22T18:55:42,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-22T18:55:42,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741887_1063 (size=12151) 2024-11-22T18:55:42,564 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/8e27507c5e0041d8afc6ca534cc7a150 2024-11-22T18:55:42,588 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/5e4ddc9d6e11499b96376034bee60261 is 50, key is test_row_0/B:col10/1732301742464/Put/seqid=0 2024-11-22T18:55:42,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:42,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1732301802591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:42,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:42,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:42,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:42,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:42,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40756 deadline: 1732301802594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:42,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:42,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:42,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:42,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:42,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40758 deadline: 1732301802595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:42,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:42,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:42,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:42,608 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-22T18:55:42,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:42,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. as already flushing 2024-11-22T18:55:42,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:42,609 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:55:42,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:55:42,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:55:42,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741888_1064 (size=12151) 2024-11-22T18:55:42,640 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/5e4ddc9d6e11499b96376034bee60261 2024-11-22T18:55:42,661 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/b16cc56652e14844a6a1f6746233f9fa is 50, key is test_row_0/C:col10/1732301742464/Put/seqid=0 2024-11-22T18:55:42,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:42,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40756 deadline: 1732301802706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:42,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:42,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:42,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:42,714 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:42,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1732301802706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:42,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:42,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:42,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:42,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:42,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40758 deadline: 1732301802709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:42,723 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:42,723 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:42,723 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:42,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741889_1065 (size=12151) 2024-11-22T18:55:42,738 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/b16cc56652e14844a6a1f6746233f9fa 2024-11-22T18:55:42,746 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/8e27507c5e0041d8afc6ca534cc7a150 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/8e27507c5e0041d8afc6ca534cc7a150 2024-11-22T18:55:42,752 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/8e27507c5e0041d8afc6ca534cc7a150, entries=150, sequenceid=251, filesize=11.9 K 2024-11-22T18:55:42,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/5e4ddc9d6e11499b96376034bee60261 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/5e4ddc9d6e11499b96376034bee60261 2024-11-22T18:55:42,760 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/5e4ddc9d6e11499b96376034bee60261, entries=150, sequenceid=251, filesize=11.9 K 2024-11-22T18:55:42,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/b16cc56652e14844a6a1f6746233f9fa as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/b16cc56652e14844a6a1f6746233f9fa 2024-11-22T18:55:42,763 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-22T18:55:42,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:42,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. as already flushing 2024-11-22T18:55:42,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:42,763 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:55:42,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:55:42,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:55:42,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-22T18:55:42,773 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/b16cc56652e14844a6a1f6746233f9fa, entries=150, sequenceid=251, filesize=11.9 K 2024-11-22T18:55:42,775 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 0fc957929e5be1c291085a12bd8cf851 in 308ms, sequenceid=251, compaction requested=true 2024-11-22T18:55:42,775 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0fc957929e5be1c291085a12bd8cf851: 2024-11-22T18:55:42,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fc957929e5be1c291085a12bd8cf851:A, priority=-2147483648, current under compaction store size is 3 2024-11-22T18:55:42,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-22T18:55:42,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fc957929e5be1c291085a12bd8cf851:B, priority=-2147483648, current under compaction store size is 3 2024-11-22T18:55:42,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-22T18:55:42,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fc957929e5be1c291085a12bd8cf851:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T18:55:42,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-22T18:55:42,919 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-22T18:55:42,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:42,920 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 0fc957929e5be1c291085a12bd8cf851 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-22T18:55:42,921 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. as already flushing 2024-11-22T18:55:42,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on 0fc957929e5be1c291085a12bd8cf851 2024-11-22T18:55:42,939 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/8c0885899fed4e0e95da32956ebe935c as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/8c0885899fed4e0e95da32956ebe935c 2024-11-22T18:55:42,941 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/3f3b4f29b00f471ab26d9d33aa8c6813 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/3f3b4f29b00f471ab26d9d33aa8c6813 2024-11-22T18:55:42,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/e437ccc403d244f196ba86b1d456b3b0 is 50, key is test_row_0/A:col10/1732301742584/Put/seqid=0 2024-11-22T18:55:42,954 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fc957929e5be1c291085a12bd8cf851/A of 0fc957929e5be1c291085a12bd8cf851 into 3f3b4f29b00f471ab26d9d33aa8c6813(size=12.4 K), total size for store is 24.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:55:42,954 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fc957929e5be1c291085a12bd8cf851: 2024-11-22T18:55:42,954 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., storeName=0fc957929e5be1c291085a12bd8cf851/A, priority=13, startTime=1732301742411; duration=0sec 2024-11-22T18:55:42,954 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-22T18:55:42,954 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fc957929e5be1c291085a12bd8cf851:A 2024-11-22T18:55:42,954 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fc957929e5be1c291085a12bd8cf851:A 2024-11-22T18:55:42,954 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T18:55:42,955 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fc957929e5be1c291085a12bd8cf851/B of 0fc957929e5be1c291085a12bd8cf851 into 8c0885899fed4e0e95da32956ebe935c(size=12.4 K), total size for store is 24.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:55:42,955 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fc957929e5be1c291085a12bd8cf851: 2024-11-22T18:55:42,955 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., storeName=0fc957929e5be1c291085a12bd8cf851/B, priority=13, startTime=1732301742411; duration=0sec 2024-11-22T18:55:42,955 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-22T18:55:42,955 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fc957929e5be1c291085a12bd8cf851:B 2024-11-22T18:55:42,955 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fc957929e5be1c291085a12bd8cf851:B 2024-11-22T18:55:42,957 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T18:55:42,957 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1541): 0fc957929e5be1c291085a12bd8cf851/C is initiating minor compaction (all files) 2024-11-22T18:55:42,957 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 4 compacting, 0 eligible, 16 blocking 2024-11-22T18:55:42,957 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fc957929e5be1c291085a12bd8cf851/C in TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:42,957 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-22T18:55:42,957 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-22T18:55:42,958 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/7d52149821a74922ae542acd73064b43, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/ca5be2fb67274703a27290081a2748ff, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/611d036604e2408ba125a29638a09507, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/b16cc56652e14844a6a1f6746233f9fa] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp, totalSize=47.9 K 2024-11-22T18:55:42,958 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. because compaction request was cancelled 2024-11-22T18:55:42,958 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fc957929e5be1c291085a12bd8cf851:C 2024-11-22T18:55:42,958 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-22T18:55:42,958 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d52149821a74922ae542acd73064b43, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732301739293 2024-11-22T18:55:42,959 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca5be2fb67274703a27290081a2748ff, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1732301739943 2024-11-22T18:55:42,959 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-22T18:55:42,959 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-22T18:55:42,959 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. because compaction request was cancelled 2024-11-22T18:55:42,960 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fc957929e5be1c291085a12bd8cf851:B 2024-11-22T18:55:42,960 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-22T18:55:42,960 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 611d036604e2408ba125a29638a09507, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1732301741176 2024-11-22T18:55:42,961 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting b16cc56652e14844a6a1f6746233f9fa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732301742458 2024-11-22T18:55:42,961 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-22T18:55:42,961 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-22T18:55:42,961 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. because compaction request was cancelled 2024-11-22T18:55:42,961 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fc957929e5be1c291085a12bd8cf851:A 2024-11-22T18:55:42,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:42,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40758 deadline: 1732301802955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:42,973 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:42,973 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:42,973 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:42,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:42,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40756 deadline: 1732301802972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:42,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:42,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:42,977 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:42,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:42,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1732301802972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:42,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:42,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:42,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:42,988 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fc957929e5be1c291085a12bd8cf851#C#compaction#54 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:55:42,988 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/6c7a55a6a84243899b33b3da989028e7 is 50, key is test_row_0/C:col10/1732301742464/Put/seqid=0 2024-11-22T18:55:42,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741890_1066 (size=12301) 2024-11-22T18:55:43,001 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/e437ccc403d244f196ba86b1d456b3b0 2024-11-22T18:55:43,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/95ca925bd5034daab5f1dc781a263c3f is 50, key is test_row_0/B:col10/1732301742584/Put/seqid=0 2024-11-22T18:55:43,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741891_1067 (size=12731) 2024-11-22T18:55:43,067 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/6c7a55a6a84243899b33b3da989028e7 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/6c7a55a6a84243899b33b3da989028e7 2024-11-22T18:55:43,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741892_1068 (size=12301) 2024-11-22T18:55:43,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-22T18:55:43,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:43,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40758 deadline: 1732301803076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:43,080 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:43,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:43,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:43,082 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0fc957929e5be1c291085a12bd8cf851/C of 0fc957929e5be1c291085a12bd8cf851 into 6c7a55a6a84243899b33b3da989028e7(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:55:43,082 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fc957929e5be1c291085a12bd8cf851: 2024-11-22T18:55:43,082 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., storeName=0fc957929e5be1c291085a12bd8cf851/C, priority=12, startTime=1732301742775; duration=0sec 2024-11-22T18:55:43,082 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:55:43,082 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fc957929e5be1c291085a12bd8cf851:C 2024-11-22T18:55:43,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:43,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40756 deadline: 1732301803085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:43,093 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:43,093 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:43,093 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:43,094 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:43,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1732301803085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:43,095 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:43,095 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:43,095 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:43,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:43,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40758 deadline: 1732301803285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:43,287 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:43,287 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:43,287 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:43,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:43,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40756 deadline: 1732301803295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:43,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:43,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:43,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:43,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:43,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1732301803306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:43,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:43,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:43,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:43,476 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/95ca925bd5034daab5f1dc781a263c3f 2024-11-22T18:55:43,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/345fccfb3b8c4a84ba8987d15d9f809a is 50, key is test_row_0/C:col10/1732301742584/Put/seqid=0 2024-11-22T18:55:43,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:43,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40782 deadline: 1732301803496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:43,500 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to d79ba0c344fb,43593,1732301724173 for 'test_row_2' in 0fc957929e5be1c291085a12bd8cf851 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] ... 32 more 2024-11-22T18:55:43,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:43,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:43,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:43,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:43,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40746 deadline: 1732301803515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:43,519 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to d79ba0c344fb,43593,1732301724173 for 'test_row_2' in 0fc957929e5be1c291085a12bd8cf851 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] ... 32 more 2024-11-22T18:55:43,519 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:43,519 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:43,520 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:43,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741893_1069 (size=12301) 2024-11-22T18:55:43,546 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/345fccfb3b8c4a84ba8987d15d9f809a 2024-11-22T18:55:43,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/e437ccc403d244f196ba86b1d456b3b0 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/e437ccc403d244f196ba86b1d456b3b0 2024-11-22T18:55:43,562 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/e437ccc403d244f196ba86b1d456b3b0, entries=150, sequenceid=278, filesize=12.0 K 2024-11-22T18:55:43,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/95ca925bd5034daab5f1dc781a263c3f as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/95ca925bd5034daab5f1dc781a263c3f 2024-11-22T18:55:43,573 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/95ca925bd5034daab5f1dc781a263c3f, entries=150, sequenceid=278, filesize=12.0 K 2024-11-22T18:55:43,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/345fccfb3b8c4a84ba8987d15d9f809a as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/345fccfb3b8c4a84ba8987d15d9f809a 2024-11-22T18:55:43,583 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/345fccfb3b8c4a84ba8987d15d9f809a, entries=150, sequenceid=278, filesize=12.0 K 2024-11-22T18:55:43,585 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=46.96 KB/48090 for 0fc957929e5be1c291085a12bd8cf851 in 665ms, sequenceid=278, compaction requested=true 2024-11-22T18:55:43,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-22T18:55:43,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 0fc957929e5be1c291085a12bd8cf851: 2024-11-22T18:55:43,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:43,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-22T18:55:43,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-22T18:55:43,595 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-22T18:55:43,595 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1360 sec 2024-11-22T18:55:43,598 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.1490 sec 2024-11-22T18:55:43,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on 0fc957929e5be1c291085a12bd8cf851 2024-11-22T18:55:43,600 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T18:55:43,601 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0fc957929e5be1c291085a12bd8cf851 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-22T18:55:43,614 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/9a303423e30946e18acb3554c9cbfd55 is 50, key is test_row_0/A:col10/1732301742927/Put/seqid=0 2024-11-22T18:55:43,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741894_1070 (size=14741) 2024-11-22T18:55:43,625 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/9a303423e30946e18acb3554c9cbfd55 2024-11-22T18:55:43,637 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/13e6881ab4e74c0d8474e53164286249 is 50, key is test_row_0/B:col10/1732301742927/Put/seqid=0 2024-11-22T18:55:43,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741895_1071 (size=12301) 2024-11-22T18:55:43,659 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/13e6881ab4e74c0d8474e53164286249 2024-11-22T18:55:43,673 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/33bd6f57c516412ca560f91b7bced824 is 50, key is test_row_0/C:col10/1732301742927/Put/seqid=0 2024-11-22T18:55:43,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:43,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1732301803672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:43,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:43,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:43,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:43,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:43,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40756 deadline: 1732301803675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:43,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:43,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40758 deadline: 1732301803676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:43,681 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:43,681 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:43,681 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:43,682 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:43,682 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:43,682 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:43,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741896_1072 (size=12301) 2024-11-22T18:55:43,691 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/33bd6f57c516412ca560f91b7bced824 2024-11-22T18:55:43,700 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/9a303423e30946e18acb3554c9cbfd55 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/9a303423e30946e18acb3554c9cbfd55 2024-11-22T18:55:43,707 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/9a303423e30946e18acb3554c9cbfd55, entries=200, sequenceid=292, filesize=14.4 K 2024-11-22T18:55:43,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/13e6881ab4e74c0d8474e53164286249 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/13e6881ab4e74c0d8474e53164286249 2024-11-22T18:55:43,718 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/13e6881ab4e74c0d8474e53164286249, entries=150, sequenceid=292, filesize=12.0 K 2024-11-22T18:55:43,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/33bd6f57c516412ca560f91b7bced824 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/33bd6f57c516412ca560f91b7bced824 2024-11-22T18:55:43,726 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/33bd6f57c516412ca560f91b7bced824, entries=150, sequenceid=292, filesize=12.0 K 2024-11-22T18:55:43,727 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 0fc957929e5be1c291085a12bd8cf851 in 127ms, sequenceid=292, compaction requested=true 2024-11-22T18:55:43,727 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0fc957929e5be1c291085a12bd8cf851: 2024-11-22T18:55:43,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fc957929e5be1c291085a12bd8cf851:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T18:55:43,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:55:43,728 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T18:55:43,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fc957929e5be1c291085a12bd8cf851:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T18:55:43,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:55:43,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fc957929e5be1c291085a12bd8cf851:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T18:55:43,728 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T18:55:43,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:55:43,729 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49450 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T18:55:43,729 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1541): 0fc957929e5be1c291085a12bd8cf851/B is initiating minor compaction (all files) 2024-11-22T18:55:43,730 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fc957929e5be1c291085a12bd8cf851/B in TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:43,730 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/8c0885899fed4e0e95da32956ebe935c, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/5e4ddc9d6e11499b96376034bee60261, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/95ca925bd5034daab5f1dc781a263c3f, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/13e6881ab4e74c0d8474e53164286249] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp, totalSize=48.3 K 2024-11-22T18:55:43,730 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51890 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T18:55:43,730 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1541): 0fc957929e5be1c291085a12bd8cf851/A is initiating minor compaction (all files) 2024-11-22T18:55:43,730 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fc957929e5be1c291085a12bd8cf851/A in TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:43,730 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c0885899fed4e0e95da32956ebe935c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1732301741176 2024-11-22T18:55:43,730 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/3f3b4f29b00f471ab26d9d33aa8c6813, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/8e27507c5e0041d8afc6ca534cc7a150, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/e437ccc403d244f196ba86b1d456b3b0, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/9a303423e30946e18acb3554c9cbfd55] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp, totalSize=50.7 K 2024-11-22T18:55:43,731 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e4ddc9d6e11499b96376034bee60261, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732301742458 2024-11-22T18:55:43,731 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f3b4f29b00f471ab26d9d33aa8c6813, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1732301741176 2024-11-22T18:55:43,732 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 95ca925bd5034daab5f1dc781a263c3f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732301742584 2024-11-22T18:55:43,732 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e27507c5e0041d8afc6ca534cc7a150, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732301742458 2024-11-22T18:55:43,732 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 13e6881ab4e74c0d8474e53164286249, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732301742927 2024-11-22T18:55:43,732 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting e437ccc403d244f196ba86b1d456b3b0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732301742584 2024-11-22T18:55:43,733 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a303423e30946e18acb3554c9cbfd55, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732301742927 2024-11-22T18:55:43,745 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fc957929e5be1c291085a12bd8cf851#B#compaction#60 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:55:43,745 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/4e70d6ac9ea749919cc1c7ec08963987 is 50, key is test_row_0/B:col10/1732301742927/Put/seqid=0 2024-11-22T18:55:43,748 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fc957929e5be1c291085a12bd8cf851#A#compaction#61 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:55:43,748 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/28442b558a6b499a9b1fb0494134c1b2 is 50, key is test_row_0/A:col10/1732301742927/Put/seqid=0 2024-11-22T18:55:43,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741898_1074 (size=12983) 2024-11-22T18:55:43,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741897_1073 (size=12983) 2024-11-22T18:55:43,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on 0fc957929e5be1c291085a12bd8cf851 2024-11-22T18:55:43,789 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T18:55:43,790 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0fc957929e5be1c291085a12bd8cf851 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-22T18:55:43,795 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/dda48163fbf0431b9a6a3d984a7b046c is 50, key is test_row_0/A:col10/1732301743673/Put/seqid=0 2024-11-22T18:55:43,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:43,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40758 deadline: 1732301803801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:43,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:43,804 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:43,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40756 deadline: 1732301803802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:43,804 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:43,804 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:43,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:43,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:43,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:43,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1732301803803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:43,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:43,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:43,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:43,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:43,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741899_1075 (size=12301) 2024-11-22T18:55:43,811 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/dda48163fbf0431b9a6a3d984a7b046c 2024-11-22T18:55:43,828 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/8877606198044f09ab3554add83069c1 is 50, key is test_row_0/B:col10/1732301743673/Put/seqid=0 2024-11-22T18:55:43,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741900_1076 (size=12301) 2024-11-22T18:55:43,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:43,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40756 deadline: 1732301803906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:43,908 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:43,908 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:43,908 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:43,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:43,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40758 deadline: 1732301803915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:43,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:43,918 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:43,918 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:43,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1732301803915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:43,918 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:43,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:43,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:43,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:44,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:44,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40756 deadline: 1732301804116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:44,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:44,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:44,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:44,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:44,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40758 deadline: 1732301804125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:44,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:44,127 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:44,128 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:44,128 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:44,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1732301804125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:44,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:44,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:44,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:44,169 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/28442b558a6b499a9b1fb0494134c1b2 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/28442b558a6b499a9b1fb0494134c1b2 2024-11-22T18:55:44,170 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/4e70d6ac9ea749919cc1c7ec08963987 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/4e70d6ac9ea749919cc1c7ec08963987 2024-11-22T18:55:44,183 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0fc957929e5be1c291085a12bd8cf851/B of 0fc957929e5be1c291085a12bd8cf851 into 4e70d6ac9ea749919cc1c7ec08963987(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:55:44,183 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0fc957929e5be1c291085a12bd8cf851/A of 0fc957929e5be1c291085a12bd8cf851 into 28442b558a6b499a9b1fb0494134c1b2(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:55:44,183 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fc957929e5be1c291085a12bd8cf851: 2024-11-22T18:55:44,183 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fc957929e5be1c291085a12bd8cf851: 2024-11-22T18:55:44,183 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., storeName=0fc957929e5be1c291085a12bd8cf851/B, priority=12, startTime=1732301743728; duration=0sec 2024-11-22T18:55:44,183 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., storeName=0fc957929e5be1c291085a12bd8cf851/A, priority=12, startTime=1732301743727; duration=0sec 2024-11-22T18:55:44,183 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:55:44,183 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fc957929e5be1c291085a12bd8cf851:A 2024-11-22T18:55:44,183 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:55:44,183 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fc957929e5be1c291085a12bd8cf851:B 2024-11-22T18:55:44,184 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T18:55:44,185 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T18:55:44,185 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1541): 0fc957929e5be1c291085a12bd8cf851/C is initiating minor compaction (all files) 2024-11-22T18:55:44,186 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fc957929e5be1c291085a12bd8cf851/C in TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:44,186 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/6c7a55a6a84243899b33b3da989028e7, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/345fccfb3b8c4a84ba8987d15d9f809a, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/33bd6f57c516412ca560f91b7bced824] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp, totalSize=36.5 K 2024-11-22T18:55:44,186 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c7a55a6a84243899b33b3da989028e7, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732301742458 2024-11-22T18:55:44,187 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 345fccfb3b8c4a84ba8987d15d9f809a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732301742584 2024-11-22T18:55:44,187 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33bd6f57c516412ca560f91b7bced824, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732301742927 2024-11-22T18:55:44,197 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fc957929e5be1c291085a12bd8cf851#C#compaction#64 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:55:44,198 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/321653053576407eb3fec52a2824f743 is 50, key is test_row_0/C:col10/1732301742927/Put/seqid=0 2024-11-22T18:55:44,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741901_1077 (size=12983) 2024-11-22T18:55:44,242 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/8877606198044f09ab3554add83069c1 2024-11-22T18:55:44,261 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/421945da95c24d9fa32f895900b2c4f9 is 50, key is test_row_0/C:col10/1732301743673/Put/seqid=0 2024-11-22T18:55:44,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741902_1078 (size=12301) 2024-11-22T18:55:44,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:44,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40756 deadline: 1732301804426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:44,428 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:44,428 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:44,428 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:44,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:44,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40758 deadline: 1732301804436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:44,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:44,441 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:44,441 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:44,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1732301804437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:44,441 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:44,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:44,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:44,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:44,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-22T18:55:44,596 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T18:55:44,597 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T18:55:44,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-22T18:55:44,600 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T18:55:44,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-22T18:55:44,602 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T18:55:44,602 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T18:55:44,613 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/321653053576407eb3fec52a2824f743 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/321653053576407eb3fec52a2824f743 2024-11-22T18:55:44,625 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fc957929e5be1c291085a12bd8cf851/C of 0fc957929e5be1c291085a12bd8cf851 into 321653053576407eb3fec52a2824f743(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:55:44,625 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fc957929e5be1c291085a12bd8cf851: 2024-11-22T18:55:44,625 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., storeName=0fc957929e5be1c291085a12bd8cf851/C, priority=13, startTime=1732301743728; duration=0sec 2024-11-22T18:55:44,626 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:55:44,626 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fc957929e5be1c291085a12bd8cf851:C 2024-11-22T18:55:44,689 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/421945da95c24d9fa32f895900b2c4f9 2024-11-22T18:55:44,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/dda48163fbf0431b9a6a3d984a7b046c as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/dda48163fbf0431b9a6a3d984a7b046c 2024-11-22T18:55:44,705 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/dda48163fbf0431b9a6a3d984a7b046c, entries=150, sequenceid=320, filesize=12.0 K 2024-11-22T18:55:44,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-22T18:55:44,707 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/8877606198044f09ab3554add83069c1 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/8877606198044f09ab3554add83069c1 2024-11-22T18:55:44,714 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/8877606198044f09ab3554add83069c1, entries=150, sequenceid=320, filesize=12.0 K 2024-11-22T18:55:44,715 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/421945da95c24d9fa32f895900b2c4f9 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/421945da95c24d9fa32f895900b2c4f9 2024-11-22T18:55:44,726 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/421945da95c24d9fa32f895900b2c4f9, entries=150, sequenceid=320, filesize=12.0 K 2024-11-22T18:55:44,728 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 0fc957929e5be1c291085a12bd8cf851 in 939ms, sequenceid=320, compaction requested=false 2024-11-22T18:55:44,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0fc957929e5be1c291085a12bd8cf851: 2024-11-22T18:55:44,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-22T18:55:44,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:44,756 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 0fc957929e5be1c291085a12bd8cf851 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-22T18:55:44,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/c30c963951ec48da8dd7b04ffdb227a2 is 50, key is test_row_0/A:col10/1732301743801/Put/seqid=0 2024-11-22T18:55:44,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741903_1079 (size=12301) 2024-11-22T18:55:44,785 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/c30c963951ec48da8dd7b04ffdb227a2 2024-11-22T18:55:44,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/c1507559441d4888868fdebc3cda9591 is 50, key is test_row_0/B:col10/1732301743801/Put/seqid=0 2024-11-22T18:55:44,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741904_1080 (size=12301) 2024-11-22T18:55:44,837 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/c1507559441d4888868fdebc3cda9591 2024-11-22T18:55:44,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/dab2debff09746439889655f2f8cca0b is 50, key is test_row_0/C:col10/1732301743801/Put/seqid=0 2024-11-22T18:55:44,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741905_1081 (size=12301) 2024-11-22T18:55:44,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-22T18:55:44,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on 0fc957929e5be1c291085a12bd8cf851 2024-11-22T18:55:44,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. as already flushing 2024-11-22T18:55:45,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:45,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40756 deadline: 1732301804999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:45,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:45,004 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:45,004 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:45,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1732301805001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:45,004 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:45,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:45,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:45,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:45,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:45,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40758 deadline: 1732301805003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:45,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:45,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:45,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:45,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:45,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1732301805116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:45,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:45,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:45,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:45,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:45,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40756 deadline: 1732301805116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:45,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:45,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:45,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:45,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40758 deadline: 1732301805117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:45,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:45,121 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:45,121 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:45,121 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:45,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-22T18:55:45,275 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/dab2debff09746439889655f2f8cca0b 2024-11-22T18:55:45,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/c30c963951ec48da8dd7b04ffdb227a2 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/c30c963951ec48da8dd7b04ffdb227a2 2024-11-22T18:55:45,289 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/c30c963951ec48da8dd7b04ffdb227a2, entries=150, sequenceid=331, filesize=12.0 K 2024-11-22T18:55:45,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/c1507559441d4888868fdebc3cda9591 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/c1507559441d4888868fdebc3cda9591 2024-11-22T18:55:45,299 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/c1507559441d4888868fdebc3cda9591, entries=150, sequenceid=331, filesize=12.0 K 2024-11-22T18:55:45,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/dab2debff09746439889655f2f8cca0b as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/dab2debff09746439889655f2f8cca0b 2024-11-22T18:55:45,307 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/dab2debff09746439889655f2f8cca0b, entries=150, sequenceid=331, filesize=12.0 K 2024-11-22T18:55:45,309 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=167.72 KB/171750 for 0fc957929e5be1c291085a12bd8cf851 in 552ms, sequenceid=331, compaction requested=true 2024-11-22T18:55:45,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 0fc957929e5be1c291085a12bd8cf851: 2024-11-22T18:55:45,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:45,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-22T18:55:45,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-22T18:55:45,313 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-22T18:55:45,313 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 709 msec 2024-11-22T18:55:45,316 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 717 msec 2024-11-22T18:55:45,329 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T18:55:45,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on 0fc957929e5be1c291085a12bd8cf851 2024-11-22T18:55:45,329 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0fc957929e5be1c291085a12bd8cf851 3/3 column families, dataSize=187.85 KB heapSize=492.94 KB 2024-11-22T18:55:45,336 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/faff56203fd949af900072b1549aa477 is 50, key is test_row_0/A:col10/1732301745327/Put/seqid=0 2024-11-22T18:55:45,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:45,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40758 deadline: 1732301805330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:45,338 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:45,339 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:45,339 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:45,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:45,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1732301805336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:45,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:45,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:45,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:45,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:45,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40756 deadline: 1732301805338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:45,342 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:45,342 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:45,342 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:45,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741906_1082 (size=12301) 2024-11-22T18:55:45,351 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/faff56203fd949af900072b1549aa477 2024-11-22T18:55:45,367 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/90e834a9d6df44cf88a4ce05b4781880 is 50, key is test_row_0/B:col10/1732301745327/Put/seqid=0 2024-11-22T18:55:45,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741907_1083 (size=12301) 2024-11-22T18:55:45,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:45,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40758 deadline: 1732301805446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:45,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:45,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:45,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40756 deadline: 1732301805446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:45,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:45,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:45,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:45,448 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:45,448 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:45,448 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:45,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1732301805446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:45,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:45,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:45,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:45,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:45,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40758 deadline: 1732301805656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:45,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:45,657 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:45,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40756 deadline: 1732301805656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:45,657 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:45,657 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:45,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:45,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:45,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:45,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1732301805656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:45,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:45,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:45,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:45,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:45,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-22T18:55:45,736 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T18:55:45,737 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T18:55:45,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-22T18:55:45,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-22T18:55:45,739 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T18:55:45,740 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T18:55:45,740 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T18:55:45,782 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/90e834a9d6df44cf88a4ce05b4781880 2024-11-22T18:55:45,793 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/49ae493c536e49298fc2faa270949f9e is 50, key is test_row_0/C:col10/1732301745327/Put/seqid=0 2024-11-22T18:55:45,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741908_1084 (size=12301) 2024-11-22T18:55:45,803 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/49ae493c536e49298fc2faa270949f9e 2024-11-22T18:55:45,809 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/faff56203fd949af900072b1549aa477 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/faff56203fd949af900072b1549aa477 2024-11-22T18:55:45,816 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/faff56203fd949af900072b1549aa477, entries=150, sequenceid=362, filesize=12.0 K 2024-11-22T18:55:45,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/90e834a9d6df44cf88a4ce05b4781880 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/90e834a9d6df44cf88a4ce05b4781880 2024-11-22T18:55:45,825 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/90e834a9d6df44cf88a4ce05b4781880, entries=150, sequenceid=362, filesize=12.0 K 2024-11-22T18:55:45,827 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/49ae493c536e49298fc2faa270949f9e as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/49ae493c536e49298fc2faa270949f9e 2024-11-22T18:55:45,834 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/49ae493c536e49298fc2faa270949f9e, entries=150, sequenceid=362, filesize=12.0 K 2024-11-22T18:55:45,835 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~187.85 KB/192360, heapSize ~492.89 KB/504720, currentSize=13.42 KB/13740 for 0fc957929e5be1c291085a12bd8cf851 in 506ms, sequenceid=362, compaction requested=true 2024-11-22T18:55:45,835 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0fc957929e5be1c291085a12bd8cf851: 2024-11-22T18:55:45,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fc957929e5be1c291085a12bd8cf851:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T18:55:45,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:55:45,835 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T18:55:45,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fc957929e5be1c291085a12bd8cf851:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T18:55:45,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:55:45,835 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T18:55:45,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fc957929e5be1c291085a12bd8cf851:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T18:55:45,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:55:45,838 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T18:55:45,838 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1541): 0fc957929e5be1c291085a12bd8cf851/A is initiating minor compaction (all files) 2024-11-22T18:55:45,838 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fc957929e5be1c291085a12bd8cf851/A in TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:45,838 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/28442b558a6b499a9b1fb0494134c1b2, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/dda48163fbf0431b9a6a3d984a7b046c, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/c30c963951ec48da8dd7b04ffdb227a2, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/faff56203fd949af900072b1549aa477] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp, totalSize=48.7 K 2024-11-22T18:55:45,839 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T18:55:45,839 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1541): 0fc957929e5be1c291085a12bd8cf851/B is initiating minor compaction (all files) 2024-11-22T18:55:45,839 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fc957929e5be1c291085a12bd8cf851/B in TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:45,839 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/4e70d6ac9ea749919cc1c7ec08963987, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/8877606198044f09ab3554add83069c1, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/c1507559441d4888868fdebc3cda9591, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/90e834a9d6df44cf88a4ce05b4781880] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp, totalSize=48.7 K 2024-11-22T18:55:45,840 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28442b558a6b499a9b1fb0494134c1b2, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732301742927 2024-11-22T18:55:45,840 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e70d6ac9ea749919cc1c7ec08963987, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732301742927 2024-11-22T18:55:45,841 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting dda48163fbf0431b9a6a3d984a7b046c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732301743672 2024-11-22T18:55:45,841 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 8877606198044f09ab3554add83069c1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732301743672 2024-11-22T18:55:45,841 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting c30c963951ec48da8dd7b04ffdb227a2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1732301743798 2024-11-22T18:55:45,842 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting faff56203fd949af900072b1549aa477, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1732301745327 2024-11-22T18:55:45,843 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting c1507559441d4888868fdebc3cda9591, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1732301743798 2024-11-22T18:55:45,844 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 90e834a9d6df44cf88a4ce05b4781880, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1732301745327 2024-11-22T18:55:45,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-22T18:55:45,855 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fc957929e5be1c291085a12bd8cf851#A#compaction#72 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:55:45,855 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/5e7d520937f94e0abdfea4ad43b2f941 is 50, key is test_row_0/A:col10/1732301745327/Put/seqid=0 2024-11-22T18:55:45,856 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fc957929e5be1c291085a12bd8cf851#B#compaction#73 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:55:45,856 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/421f01379dd843c880784f6d24d62a22 is 50, key is test_row_0/B:col10/1732301745327/Put/seqid=0 2024-11-22T18:55:45,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741909_1085 (size=13119) 2024-11-22T18:55:45,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741910_1086 (size=13119) 2024-11-22T18:55:45,873 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/421f01379dd843c880784f6d24d62a22 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/421f01379dd843c880784f6d24d62a22 2024-11-22T18:55:45,879 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0fc957929e5be1c291085a12bd8cf851/B of 0fc957929e5be1c291085a12bd8cf851 into 421f01379dd843c880784f6d24d62a22(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:55:45,879 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fc957929e5be1c291085a12bd8cf851: 2024-11-22T18:55:45,879 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., storeName=0fc957929e5be1c291085a12bd8cf851/B, priority=12, startTime=1732301745835; duration=0sec 2024-11-22T18:55:45,880 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:55:45,880 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fc957929e5be1c291085a12bd8cf851:B 2024-11-22T18:55:45,880 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T18:55:45,884 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T18:55:45,884 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1541): 0fc957929e5be1c291085a12bd8cf851/C is initiating minor compaction (all files) 2024-11-22T18:55:45,884 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fc957929e5be1c291085a12bd8cf851/C in TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:45,884 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/321653053576407eb3fec52a2824f743, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/421945da95c24d9fa32f895900b2c4f9, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/dab2debff09746439889655f2f8cca0b, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/49ae493c536e49298fc2faa270949f9e] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp, totalSize=48.7 K 2024-11-22T18:55:45,884 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 321653053576407eb3fec52a2824f743, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732301742927 2024-11-22T18:55:45,885 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 421945da95c24d9fa32f895900b2c4f9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732301743672 2024-11-22T18:55:45,885 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting dab2debff09746439889655f2f8cca0b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1732301743798 2024-11-22T18:55:45,886 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 49ae493c536e49298fc2faa270949f9e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1732301745327 2024-11-22T18:55:45,893 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-22T18:55:45,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:45,894 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 0fc957929e5be1c291085a12bd8cf851 3/3 column families, dataSize=13.42 KB heapSize=35.91 KB 2024-11-22T18:55:45,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/83e6fb16e6fe49d88a8fa38bacabc2d3 is 50, key is test_row_0/A:col10/1732301745335/Put/seqid=0 2024-11-22T18:55:45,906 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fc957929e5be1c291085a12bd8cf851#C#compaction#75 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:55:45,906 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/5918205efb034fc886836d11e301b29b is 50, key is test_row_0/C:col10/1732301745327/Put/seqid=0 2024-11-22T18:55:45,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741911_1087 (size=7415) 2024-11-22T18:55:45,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741912_1088 (size=13119) 2024-11-22T18:55:45,966 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/5918205efb034fc886836d11e301b29b as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/5918205efb034fc886836d11e301b29b 2024-11-22T18:55:45,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on 0fc957929e5be1c291085a12bd8cf851 2024-11-22T18:55:45,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. as already flushing 2024-11-22T18:55:45,979 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0fc957929e5be1c291085a12bd8cf851/C of 0fc957929e5be1c291085a12bd8cf851 into 5918205efb034fc886836d11e301b29b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:55:45,979 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fc957929e5be1c291085a12bd8cf851: 2024-11-22T18:55:45,979 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., storeName=0fc957929e5be1c291085a12bd8cf851/C, priority=12, startTime=1732301745835; duration=0sec 2024-11-22T18:55:45,979 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:55:45,979 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fc957929e5be1c291085a12bd8cf851:C 2024-11-22T18:55:46,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:46,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:46,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40756 deadline: 1732301806050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:46,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1732301806051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:46,055 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:46,055 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:46,055 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:46,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:46,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-22T18:55:46,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:46,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:46,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:46,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40758 deadline: 1732301806055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:46,058 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:46,058 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:46,058 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:46,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:46,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:46,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1732301806165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:46,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40756 deadline: 1732301806165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:46,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:46,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:46,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:46,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40758 deadline: 1732301806165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:46,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:46,168 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:46,168 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:46,168 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:46,168 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:46,168 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:46,168 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:46,273 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/5e7d520937f94e0abdfea4ad43b2f941 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/5e7d520937f94e0abdfea4ad43b2f941 2024-11-22T18:55:46,280 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0fc957929e5be1c291085a12bd8cf851/A of 0fc957929e5be1c291085a12bd8cf851 into 5e7d520937f94e0abdfea4ad43b2f941(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:55:46,280 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fc957929e5be1c291085a12bd8cf851: 2024-11-22T18:55:46,280 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., storeName=0fc957929e5be1c291085a12bd8cf851/A, priority=12, startTime=1732301745835; duration=0sec 2024-11-22T18:55:46,280 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:55:46,280 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fc957929e5be1c291085a12bd8cf851:A 2024-11-22T18:55:46,336 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/83e6fb16e6fe49d88a8fa38bacabc2d3 2024-11-22T18:55:46,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/980677100bb34d83aa3fd8a34c8d6356 is 50, key is test_row_0/B:col10/1732301745335/Put/seqid=0 2024-11-22T18:55:46,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741913_1089 (size=7415) 2024-11-22T18:55:46,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-22T18:55:46,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:46,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1732301806375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:46,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:46,377 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:46,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40758 deadline: 1732301806375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:46,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:46,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:46,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:46,378 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:46,378 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:46,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40756 deadline: 1732301806375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:46,378 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:46,378 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:46,378 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:46,378 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:46,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:46,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1732301806686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:46,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:46,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:46,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:46,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:46,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40758 deadline: 1732301806686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:46,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:46,688 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:46,688 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:46,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40756 deadline: 1732301806686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:46,688 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:46,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:46,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:46,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:46,754 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/980677100bb34d83aa3fd8a34c8d6356 2024-11-22T18:55:46,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/f3303a5db0f34ef588e9b2968691210a is 50, key is test_row_0/C:col10/1732301745335/Put/seqid=0 2024-11-22T18:55:46,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741914_1090 (size=7415) 2024-11-22T18:55:46,770 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/f3303a5db0f34ef588e9b2968691210a 2024-11-22T18:55:46,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/83e6fb16e6fe49d88a8fa38bacabc2d3 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/83e6fb16e6fe49d88a8fa38bacabc2d3 2024-11-22T18:55:46,782 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/83e6fb16e6fe49d88a8fa38bacabc2d3, entries=50, sequenceid=368, filesize=7.2 K 2024-11-22T18:55:46,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/980677100bb34d83aa3fd8a34c8d6356 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/980677100bb34d83aa3fd8a34c8d6356 2024-11-22T18:55:46,788 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/980677100bb34d83aa3fd8a34c8d6356, entries=50, sequenceid=368, filesize=7.2 K 2024-11-22T18:55:46,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/f3303a5db0f34ef588e9b2968691210a as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/f3303a5db0f34ef588e9b2968691210a 2024-11-22T18:55:46,794 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/f3303a5db0f34ef588e9b2968691210a, entries=50, sequenceid=368, filesize=7.2 K 2024-11-22T18:55:46,795 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=187.85 KB/192360 for 0fc957929e5be1c291085a12bd8cf851 in 901ms, sequenceid=368, compaction requested=false 2024-11-22T18:55:46,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 0fc957929e5be1c291085a12bd8cf851: 2024-11-22T18:55:46,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:46,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-22T18:55:46,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-22T18:55:46,799 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-22T18:55:46,799 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0570 sec 2024-11-22T18:55:46,801 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.0620 sec 2024-11-22T18:55:46,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-22T18:55:46,876 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T18:55:46,877 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T18:55:46,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-22T18:55:46,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-22T18:55:46,880 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T18:55:46,881 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T18:55:46,881 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T18:55:46,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-22T18:55:47,035 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-22T18:55:47,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:47,036 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing 0fc957929e5be1c291085a12bd8cf851 3/3 column families, dataSize=187.85 KB heapSize=492.94 KB 2024-11-22T18:55:47,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/f4a7a6e0a8be41f79004bb940cf7c45a is 50, key is test_row_0/A:col10/1732301746043/Put/seqid=0 2024-11-22T18:55:47,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741915_1091 (size=12301) 2024-11-22T18:55:47,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-22T18:55:47,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on 0fc957929e5be1c291085a12bd8cf851 2024-11-22T18:55:47,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. as already flushing 2024-11-22T18:55:47,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:47,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1732301807205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:47,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:47,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:47,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:47,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:47,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40756 deadline: 1732301807206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:47,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:47,212 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:47,212 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:47,212 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:47,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40758 deadline: 1732301807206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:47,212 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:47,212 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:47,212 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:47,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:47,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1732301807315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:47,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:47,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:47,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:47,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:47,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40758 deadline: 1732301807315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:47,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:47,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40756 deadline: 1732301807316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:47,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:47,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:47,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:47,322 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:47,322 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:47,322 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:47,501 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/f4a7a6e0a8be41f79004bb940cf7c45a 2024-11-22T18:55:47,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-22T18:55:47,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:47,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40782 deadline: 1732301807505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:47,507 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to d79ba0c344fb,43593,1732301724173 for 'test_row_2' in 0fc957929e5be1c291085a12bd8cf851 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] ... 32 more 2024-11-22T18:55:47,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:47,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:47,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:47,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/b382dea8320c4bf1b65d4e6459785d87 is 50, key is test_row_0/B:col10/1732301746043/Put/seqid=0 2024-11-22T18:55:47,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:47,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1732301807526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:47,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:47,528 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:47,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40758 deadline: 1732301807526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:47,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:47,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:47,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:47,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40756 deadline: 1732301807526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:47,529 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:47,529 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:47,529 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:47,529 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:47,529 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:47,529 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:47,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741916_1092 (size=12301) 2024-11-22T18:55:47,547 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/b382dea8320c4bf1b65d4e6459785d87 2024-11-22T18:55:47,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:47,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40746 deadline: 1732301807545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:47,548 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to d79ba0c344fb,43593,1732301724173 for 'test_row_2' in 0fc957929e5be1c291085a12bd8cf851 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8242 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] ... 32 more 2024-11-22T18:55:47,549 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:47,549 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:47,549 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:47,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/c9db5bebfce640a9a3aa72c445678da2 is 50, key is test_row_0/C:col10/1732301746043/Put/seqid=0 2024-11-22T18:55:47,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741917_1093 (size=12301) 2024-11-22T18:55:47,576 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/c9db5bebfce640a9a3aa72c445678da2 2024-11-22T18:55:47,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/f4a7a6e0a8be41f79004bb940cf7c45a as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/f4a7a6e0a8be41f79004bb940cf7c45a 2024-11-22T18:55:47,597 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/f4a7a6e0a8be41f79004bb940cf7c45a, entries=150, sequenceid=401, filesize=12.0 K 2024-11-22T18:55:47,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/b382dea8320c4bf1b65d4e6459785d87 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/b382dea8320c4bf1b65d4e6459785d87 2024-11-22T18:55:47,604 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/b382dea8320c4bf1b65d4e6459785d87, entries=150, sequenceid=401, filesize=12.0 K 2024-11-22T18:55:47,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/c9db5bebfce640a9a3aa72c445678da2 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/c9db5bebfce640a9a3aa72c445678da2 2024-11-22T18:55:47,617 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/c9db5bebfce640a9a3aa72c445678da2, entries=150, sequenceid=401, filesize=12.0 K 2024-11-22T18:55:47,618 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~187.85 KB/192360, heapSize ~492.89 KB/504720, currentSize=20.13 KB/20610 for 0fc957929e5be1c291085a12bd8cf851 in 582ms, sequenceid=401, compaction requested=true 2024-11-22T18:55:47,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for 0fc957929e5be1c291085a12bd8cf851: 2024-11-22T18:55:47,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:47,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-22T18:55:47,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-22T18:55:47,623 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-22T18:55:47,623 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 740 msec 2024-11-22T18:55:47,626 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 747 msec 2024-11-22T18:55:47,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on 0fc957929e5be1c291085a12bd8cf851 2024-11-22T18:55:47,850 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T18:55:47,851 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0fc957929e5be1c291085a12bd8cf851 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-22T18:55:47,859 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/ae5e927aa37d4010b6b6fa0b888c5981 is 50, key is test_row_0/A:col10/1732301747838/Put/seqid=0 2024-11-22T18:55:47,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741918_1094 (size=12301) 2024-11-22T18:55:47,890 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/ae5e927aa37d4010b6b6fa0b888c5981 2024-11-22T18:55:47,908 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/d3787c062ca1433095df13b8206ce4d4 is 50, key is test_row_0/B:col10/1732301747838/Put/seqid=0 2024-11-22T18:55:47,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741919_1095 (size=12301) 2024-11-22T18:55:47,962 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/d3787c062ca1433095df13b8206ce4d4 2024-11-22T18:55:47,975 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/2b298136f05e409d96e3c937680223db is 50, key is test_row_0/C:col10/1732301747838/Put/seqid=0 2024-11-22T18:55:47,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:47,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40756 deadline: 1732301807978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:47,987 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:47,987 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:47,987 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:48,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:48,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40758 deadline: 1732301807993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:48,003 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:48,003 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:48,003 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:48,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:48,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1732301807993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:48,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741920_1096 (size=12301) 2024-11-22T18:55:48,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:48,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:48,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:48,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-22T18:55:48,016 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T18:55:48,017 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/2b298136f05e409d96e3c937680223db 2024-11-22T18:55:48,017 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T18:55:48,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees 2024-11-22T18:55:48,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-22T18:55:48,021 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T18:55:48,022 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T18:55:48,022 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T18:55:48,026 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/ae5e927aa37d4010b6b6fa0b888c5981 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/ae5e927aa37d4010b6b6fa0b888c5981 2024-11-22T18:55:48,033 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/ae5e927aa37d4010b6b6fa0b888c5981, entries=150, sequenceid=412, filesize=12.0 K 2024-11-22T18:55:48,035 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/d3787c062ca1433095df13b8206ce4d4 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/d3787c062ca1433095df13b8206ce4d4 2024-11-22T18:55:48,041 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/d3787c062ca1433095df13b8206ce4d4, entries=150, sequenceid=412, filesize=12.0 K 2024-11-22T18:55:48,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/2b298136f05e409d96e3c937680223db as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/2b298136f05e409d96e3c937680223db 2024-11-22T18:55:48,049 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/2b298136f05e409d96e3c937680223db, entries=150, sequenceid=412, filesize=12.0 K 2024-11-22T18:55:48,050 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 0fc957929e5be1c291085a12bd8cf851 in 200ms, sequenceid=412, compaction requested=true 2024-11-22T18:55:48,050 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0fc957929e5be1c291085a12bd8cf851: 2024-11-22T18:55:48,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fc957929e5be1c291085a12bd8cf851:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T18:55:48,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:55:48,050 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T18:55:48,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fc957929e5be1c291085a12bd8cf851:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T18:55:48,050 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T18:55:48,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:55:48,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fc957929e5be1c291085a12bd8cf851:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T18:55:48,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:55:48,053 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45136 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T18:55:48,053 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1541): 0fc957929e5be1c291085a12bd8cf851/A is initiating minor compaction (all files) 2024-11-22T18:55:48,053 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fc957929e5be1c291085a12bd8cf851/A in TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:48,053 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/5e7d520937f94e0abdfea4ad43b2f941, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/83e6fb16e6fe49d88a8fa38bacabc2d3, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/f4a7a6e0a8be41f79004bb940cf7c45a, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/ae5e927aa37d4010b6b6fa0b888c5981] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp, totalSize=44.1 K 2024-11-22T18:55:48,054 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45136 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T18:55:48,054 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1541): 0fc957929e5be1c291085a12bd8cf851/B is initiating minor compaction (all files) 2024-11-22T18:55:48,054 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fc957929e5be1c291085a12bd8cf851/B in TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:48,054 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/421f01379dd843c880784f6d24d62a22, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/980677100bb34d83aa3fd8a34c8d6356, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/b382dea8320c4bf1b65d4e6459785d87, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/d3787c062ca1433095df13b8206ce4d4] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp, totalSize=44.1 K 2024-11-22T18:55:48,054 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e7d520937f94e0abdfea4ad43b2f941, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1732301745327 2024-11-22T18:55:48,055 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 421f01379dd843c880784f6d24d62a22, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1732301745327 2024-11-22T18:55:48,055 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83e6fb16e6fe49d88a8fa38bacabc2d3, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1732301745335 2024-11-22T18:55:48,055 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 980677100bb34d83aa3fd8a34c8d6356, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1732301745335 2024-11-22T18:55:48,055 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4a7a6e0a8be41f79004bb940cf7c45a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1732301746043 2024-11-22T18:55:48,056 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting b382dea8320c4bf1b65d4e6459785d87, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1732301746043 2024-11-22T18:55:48,056 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae5e927aa37d4010b6b6fa0b888c5981, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1732301747838 2024-11-22T18:55:48,057 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting d3787c062ca1433095df13b8206ce4d4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1732301747838 2024-11-22T18:55:48,077 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fc957929e5be1c291085a12bd8cf851#B#compaction#84 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:55:48,078 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/2863e5527ad8424fbab573bb31ee50d0 is 50, key is test_row_0/B:col10/1732301747838/Put/seqid=0 2024-11-22T18:55:48,087 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fc957929e5be1c291085a12bd8cf851#A#compaction#85 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:55:48,087 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/56cd757edd5041bb8ed7ff66ae945d31 is 50, key is test_row_0/A:col10/1732301747838/Put/seqid=0 2024-11-22T18:55:48,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on 0fc957929e5be1c291085a12bd8cf851 2024-11-22T18:55:48,101 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T18:55:48,101 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0fc957929e5be1c291085a12bd8cf851 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-22T18:55:48,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741921_1097 (size=13255) 2024-11-22T18:55:48,112 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/2863e5527ad8424fbab573bb31ee50d0 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/2863e5527ad8424fbab573bb31ee50d0 2024-11-22T18:55:48,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741922_1098 (size=13255) 2024-11-22T18:55:48,116 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/42cbbe96e24548f9b122096d0d2d64e3 is 50, key is test_row_0/A:col10/1732301748099/Put/seqid=0 2024-11-22T18:55:48,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:48,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40756 deadline: 1732301808118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:48,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:48,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:48,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:48,122 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0fc957929e5be1c291085a12bd8cf851/B of 0fc957929e5be1c291085a12bd8cf851 into 2863e5527ad8424fbab573bb31ee50d0(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:55:48,122 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fc957929e5be1c291085a12bd8cf851: 2024-11-22T18:55:48,122 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., storeName=0fc957929e5be1c291085a12bd8cf851/B, priority=12, startTime=1732301748050; duration=0sec 2024-11-22T18:55:48,122 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:55:48,122 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fc957929e5be1c291085a12bd8cf851:B 2024-11-22T18:55:48,122 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T18:55:48,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:48,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40758 deadline: 1732301808120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:48,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:48,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1732301808121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:48,124 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45136 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T18:55:48,124 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1541): 0fc957929e5be1c291085a12bd8cf851/C is initiating minor compaction (all files) 2024-11-22T18:55:48,124 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fc957929e5be1c291085a12bd8cf851/C in TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:48,124 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/5918205efb034fc886836d11e301b29b, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/f3303a5db0f34ef588e9b2968691210a, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/c9db5bebfce640a9a3aa72c445678da2, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/2b298136f05e409d96e3c937680223db] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp, totalSize=44.1 K 2024-11-22T18:55:48,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:48,124 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:48,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:48,124 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:48,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:48,124 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:48,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-22T18:55:48,127 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 5918205efb034fc886836d11e301b29b, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1732301745327 2024-11-22T18:55:48,127 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting f3303a5db0f34ef588e9b2968691210a, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1732301745335 2024-11-22T18:55:48,128 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting c9db5bebfce640a9a3aa72c445678da2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1732301746043 2024-11-22T18:55:48,129 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b298136f05e409d96e3c937680223db, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1732301747838 2024-11-22T18:55:48,131 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/56cd757edd5041bb8ed7ff66ae945d31 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/56cd757edd5041bb8ed7ff66ae945d31 2024-11-22T18:55:48,141 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0fc957929e5be1c291085a12bd8cf851/A of 0fc957929e5be1c291085a12bd8cf851 into 56cd757edd5041bb8ed7ff66ae945d31(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:55:48,141 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fc957929e5be1c291085a12bd8cf851: 2024-11-22T18:55:48,141 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., storeName=0fc957929e5be1c291085a12bd8cf851/A, priority=12, startTime=1732301748050; duration=0sec 2024-11-22T18:55:48,141 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:55:48,141 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fc957929e5be1c291085a12bd8cf851:A 2024-11-22T18:55:48,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741923_1099 (size=14741) 2024-11-22T18:55:48,155 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/42cbbe96e24548f9b122096d0d2d64e3 2024-11-22T18:55:48,160 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fc957929e5be1c291085a12bd8cf851#C#compaction#87 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:55:48,161 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/e0dc9a30462545ff9dd37b2cc6fcb802 is 50, key is test_row_0/C:col10/1732301747838/Put/seqid=0 2024-11-22T18:55:48,172 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/820219ead312435f9b4c32193505849f is 50, key is test_row_0/B:col10/1732301748099/Put/seqid=0 2024-11-22T18:55:48,175 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-22T18:55:48,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:48,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. as already flushing 2024-11-22T18:55:48,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:48,176 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:55:48,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:55:48,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:55:48,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741924_1100 (size=13255) 2024-11-22T18:55:48,203 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/e0dc9a30462545ff9dd37b2cc6fcb802 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/e0dc9a30462545ff9dd37b2cc6fcb802 2024-11-22T18:55:48,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741925_1101 (size=12301) 2024-11-22T18:55:48,208 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/820219ead312435f9b4c32193505849f 2024-11-22T18:55:48,215 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0fc957929e5be1c291085a12bd8cf851/C of 0fc957929e5be1c291085a12bd8cf851 into e0dc9a30462545ff9dd37b2cc6fcb802(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:55:48,216 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fc957929e5be1c291085a12bd8cf851: 2024-11-22T18:55:48,216 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., storeName=0fc957929e5be1c291085a12bd8cf851/C, priority=12, startTime=1732301748051; duration=0sec 2024-11-22T18:55:48,216 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:55:48,216 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fc957929e5be1c291085a12bd8cf851:C 2024-11-22T18:55:48,224 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/79515d36d4ae4846896a0592c1b05cc0 is 50, key is test_row_0/C:col10/1732301748099/Put/seqid=0 2024-11-22T18:55:48,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:48,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40756 deadline: 1732301808225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:48,226 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:48,226 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:48,227 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:48,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741926_1102 (size=12301) 2024-11-22T18:55:48,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:48,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:48,239 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/79515d36d4ae4846896a0592c1b05cc0 2024-11-22T18:55:48,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1732301808236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:48,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40758 deadline: 1732301808236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:48,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:48,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:48,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:48,240 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:48,240 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:48,240 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:48,249 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/42cbbe96e24548f9b122096d0d2d64e3 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/42cbbe96e24548f9b122096d0d2d64e3 2024-11-22T18:55:48,257 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/42cbbe96e24548f9b122096d0d2d64e3, entries=200, sequenceid=439, filesize=14.4 K 2024-11-22T18:55:48,259 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/820219ead312435f9b4c32193505849f as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/820219ead312435f9b4c32193505849f 2024-11-22T18:55:48,267 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/820219ead312435f9b4c32193505849f, entries=150, sequenceid=439, filesize=12.0 K 2024-11-22T18:55:48,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/79515d36d4ae4846896a0592c1b05cc0 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/79515d36d4ae4846896a0592c1b05cc0 2024-11-22T18:55:48,279 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/79515d36d4ae4846896a0592c1b05cc0, entries=150, sequenceid=439, filesize=12.0 K 2024-11-22T18:55:48,280 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 0fc957929e5be1c291085a12bd8cf851 in 179ms, sequenceid=439, compaction requested=false 2024-11-22T18:55:48,280 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0fc957929e5be1c291085a12bd8cf851: 2024-11-22T18:55:48,334 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-22T18:55:48,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:48,335 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2902): Flushing 0fc957929e5be1c291085a12bd8cf851 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-22T18:55:48,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-22T18:55:48,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/44aae49019ab4da1be7a624deb925112 is 50, key is test_row_0/A:col10/1732301748117/Put/seqid=0 2024-11-22T18:55:48,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741927_1103 (size=9857) 2024-11-22T18:55:48,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on 0fc957929e5be1c291085a12bd8cf851 2024-11-22T18:55:48,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. as already flushing 2024-11-22T18:55:48,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:48,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1732301808484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:48,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:48,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:48,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:48,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40756 deadline: 1732301808484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:48,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:48,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:48,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:48,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:48,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:48,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40758 deadline: 1732301808489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:48,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:48,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:48,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:48,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:48,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1732301808595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:48,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:48,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:48,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40756 deadline: 1732301808595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:48,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:48,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:48,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:48,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40758 deadline: 1732301808595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:48,599 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:48,599 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:48,599 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:48,600 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:48,600 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:48,600 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:48,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-22T18:55:48,747 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/44aae49019ab4da1be7a624deb925112 2024-11-22T18:55:48,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/efc4e2d505fc4f009568798607ea9e03 is 50, key is test_row_0/B:col10/1732301748117/Put/seqid=0 2024-11-22T18:55:48,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741928_1104 (size=9857) 2024-11-22T18:55:48,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:48,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1732301808805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:48,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:48,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:48,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:48,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40758 deadline: 1732301808805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:48,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:48,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:48,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:48,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40756 deadline: 1732301808805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:48,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:48,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:48,809 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:48,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:48,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:49,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:49,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1732301809116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:49,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:49,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:49,118 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:49,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40758 deadline: 1732301809116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:49,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40756 deadline: 1732301809116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:49,118 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:49,118 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:49,118 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:49,118 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:49,118 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:49,120 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:49,120 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:49,120 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:49,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-22T18:55:49,162 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/efc4e2d505fc4f009568798607ea9e03 2024-11-22T18:55:49,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/eb9df5a862154ff381b809bc01521f43 is 50, key is test_row_0/C:col10/1732301748117/Put/seqid=0 2024-11-22T18:55:49,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741929_1105 (size=9857) 2024-11-22T18:55:49,579 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/eb9df5a862154ff381b809bc01521f43 2024-11-22T18:55:49,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/44aae49019ab4da1be7a624deb925112 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/44aae49019ab4da1be7a624deb925112 2024-11-22T18:55:49,593 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/44aae49019ab4da1be7a624deb925112, entries=100, sequenceid=451, filesize=9.6 K 2024-11-22T18:55:49,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/efc4e2d505fc4f009568798607ea9e03 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/efc4e2d505fc4f009568798607ea9e03 2024-11-22T18:55:49,599 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/efc4e2d505fc4f009568798607ea9e03, entries=100, sequenceid=451, filesize=9.6 K 2024-11-22T18:55:49,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/eb9df5a862154ff381b809bc01521f43 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/eb9df5a862154ff381b809bc01521f43 2024-11-22T18:55:49,606 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/eb9df5a862154ff381b809bc01521f43, entries=100, sequenceid=451, filesize=9.6 K 2024-11-22T18:55:49,607 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for 0fc957929e5be1c291085a12bd8cf851 in 1273ms, sequenceid=451, compaction requested=true 2024-11-22T18:55:49,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2603): Flush status journal for 0fc957929e5be1c291085a12bd8cf851: 2024-11-22T18:55:49,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:49,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=28 2024-11-22T18:55:49,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster(4169): Remote procedure done, pid=28 2024-11-22T18:55:49,611 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-22T18:55:49,612 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5870 sec 2024-11-22T18:55:49,614 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees in 1.5950 sec 2024-11-22T18:55:49,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on 0fc957929e5be1c291085a12bd8cf851 2024-11-22T18:55:49,629 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T18:55:49,630 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0fc957929e5be1c291085a12bd8cf851 3/3 column families, dataSize=181.14 KB heapSize=475.36 KB 2024-11-22T18:55:49,637 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/afd43bb1783741c6a2c6c28df29fc588 is 50, key is test_row_0/A:col10/1732301749627/Put/seqid=0 2024-11-22T18:55:49,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:49,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40756 deadline: 1732301809639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:49,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:49,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1732301809641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:49,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:49,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:49,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:49,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:49,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:49,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:49,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:49,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40758 deadline: 1732301809642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:49,643 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:49,643 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:49,643 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:49,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741930_1106 (size=14741) 2024-11-22T18:55:49,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:49,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1732301809745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:49,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:49,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:49,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:49,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:49,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40758 deadline: 1732301809746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:49,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:49,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40756 deadline: 1732301809746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:49,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:49,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:49,751 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:49,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:49,751 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:49,751 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:49,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:49,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1732301809955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:49,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:49,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:49,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:49,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40756 deadline: 1732301809955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:49,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:49,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:49,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:49,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40758 deadline: 1732301809955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:49,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:49,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:49,959 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:49,959 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:49,959 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:50,047 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=481 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/afd43bb1783741c6a2c6c28df29fc588 2024-11-22T18:55:50,055 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/93f7b603ffcb4352a1a015e85076f3a0 is 50, key is test_row_0/B:col10/1732301749627/Put/seqid=0 2024-11-22T18:55:50,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741931_1107 (size=12301) 2024-11-22T18:55:50,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-22T18:55:50,165 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T18:55:50,166 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T18:55:50,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees 2024-11-22T18:55:50,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-22T18:55:50,168 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=29, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T18:55:50,169 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=29, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T18:55:50,169 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T18:55:50,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:50,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1732301810266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:50,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:50,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:50,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:50,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40756 deadline: 1732301810266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:50,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:50,267 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:50,267 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:50,267 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:50,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:50,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40758 deadline: 1732301810266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:50,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:50,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:50,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:50,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-22T18:55:50,322 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-22T18:55:50,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:50,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. as already flushing 2024-11-22T18:55:50,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:50,322 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:55:50,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:55:50,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:55:50,460 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=481 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/93f7b603ffcb4352a1a015e85076f3a0 2024-11-22T18:55:50,471 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/68fb4e6e0a6b4c90863f63afd8da13c9 is 50, key is test_row_0/C:col10/1732301749627/Put/seqid=0 2024-11-22T18:55:50,475 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-22T18:55:50,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:50,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. as already flushing 2024-11-22T18:55:50,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:50,476 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:55:50,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:55:50,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:55:50,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741932_1108 (size=12301) 2024-11-22T18:55:50,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-22T18:55:50,508 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-22T18:55:50,508 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-22T18:55:50,508 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T18:55:50,508 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T18:55:50,511 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-22T18:55:50,511 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T18:55:50,514 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-22T18:55:50,514 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T18:55:50,514 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:55:50,514 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:55:50,514 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:55:50,514 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:55:50,515 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:55:50,515 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:55:50,515 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:55:50,515 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:55:50,515 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T18:55:50,515 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T18:55:50,515 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T18:55:50,515 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T18:55:50,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-22T18:55:50,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:50,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. as already flushing 2024-11-22T18:55:50,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:50,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:55:50,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:55:50,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:55:50,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:50,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:50,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:55:50,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40764 deadline: 1732301810776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:50,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40758 deadline: 1732301810776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:50,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40756 deadline: 1732301810776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:50,777 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:50,777 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:50,777 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:50,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:50,777 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:50,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:50,777 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:55:50,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:50,777 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:55:50,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-22T18:55:50,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:50,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. as already flushing 2024-11-22T18:55:50,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:50,782 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:55:50,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:55:50,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:55:50,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-22T18:55:50,887 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=481 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/68fb4e6e0a6b4c90863f63afd8da13c9 2024-11-22T18:55:50,892 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/afd43bb1783741c6a2c6c28df29fc588 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/afd43bb1783741c6a2c6c28df29fc588 2024-11-22T18:55:50,896 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/afd43bb1783741c6a2c6c28df29fc588, entries=200, sequenceid=481, filesize=14.4 K 2024-11-22T18:55:50,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/93f7b603ffcb4352a1a015e85076f3a0 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/93f7b603ffcb4352a1a015e85076f3a0 2024-11-22T18:55:50,900 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/93f7b603ffcb4352a1a015e85076f3a0, entries=150, sequenceid=481, filesize=12.0 K 2024-11-22T18:55:50,901 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/68fb4e6e0a6b4c90863f63afd8da13c9 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/68fb4e6e0a6b4c90863f63afd8da13c9 2024-11-22T18:55:50,905 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/68fb4e6e0a6b4c90863f63afd8da13c9, entries=150, sequenceid=481, filesize=12.0 K 2024-11-22T18:55:50,906 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~181.14 KB/185490, heapSize ~475.31 KB/486720, currentSize=20.13 KB/20610 for 0fc957929e5be1c291085a12bd8cf851 in 1276ms, sequenceid=481, compaction requested=true 2024-11-22T18:55:50,906 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0fc957929e5be1c291085a12bd8cf851: 2024-11-22T18:55:50,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fc957929e5be1c291085a12bd8cf851:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T18:55:50,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:55:50,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fc957929e5be1c291085a12bd8cf851:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T18:55:50,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:55:50,906 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T18:55:50,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fc957929e5be1c291085a12bd8cf851:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T18:55:50,906 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T18:55:50,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:55:50,907 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47714 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T18:55:50,908 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52594 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T18:55:50,908 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1541): 0fc957929e5be1c291085a12bd8cf851/B is initiating minor compaction (all files) 2024-11-22T18:55:50,908 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1541): 0fc957929e5be1c291085a12bd8cf851/A is initiating minor compaction (all files) 2024-11-22T18:55:50,908 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fc957929e5be1c291085a12bd8cf851/B in TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:50,908 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fc957929e5be1c291085a12bd8cf851/A in TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:50,908 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/2863e5527ad8424fbab573bb31ee50d0, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/820219ead312435f9b4c32193505849f, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/efc4e2d505fc4f009568798607ea9e03, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/93f7b603ffcb4352a1a015e85076f3a0] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp, totalSize=46.6 K 2024-11-22T18:55:50,908 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/56cd757edd5041bb8ed7ff66ae945d31, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/42cbbe96e24548f9b122096d0d2d64e3, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/44aae49019ab4da1be7a624deb925112, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/afd43bb1783741c6a2c6c28df29fc588] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp, totalSize=51.4 K 2024-11-22T18:55:50,908 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 2863e5527ad8424fbab573bb31ee50d0, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1732301747838 2024-11-22T18:55:50,908 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 56cd757edd5041bb8ed7ff66ae945d31, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1732301747838 2024-11-22T18:55:50,909 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 820219ead312435f9b4c32193505849f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1732301747981 2024-11-22T18:55:50,909 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42cbbe96e24548f9b122096d0d2d64e3, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1732301747908 2024-11-22T18:55:50,909 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting efc4e2d505fc4f009568798607ea9e03, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1732301748117 2024-11-22T18:55:50,909 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44aae49019ab4da1be7a624deb925112, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1732301748117 2024-11-22T18:55:50,909 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 93f7b603ffcb4352a1a015e85076f3a0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1732301748482 2024-11-22T18:55:50,909 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting afd43bb1783741c6a2c6c28df29fc588, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1732301748482 2024-11-22T18:55:50,918 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fc957929e5be1c291085a12bd8cf851#B#compaction#96 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:55:50,918 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fc957929e5be1c291085a12bd8cf851#A#compaction#97 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:55:50,918 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/cd1dd6b9f2164454bc49620caec46e6e is 50, key is test_row_0/B:col10/1732301749627/Put/seqid=0 2024-11-22T18:55:50,919 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/504de3f8256c4bcda61007c6106b9691 is 50, key is test_row_0/A:col10/1732301749627/Put/seqid=0 2024-11-22T18:55:50,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741933_1109 (size=13391) 2024-11-22T18:55:50,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741934_1110 (size=13391) 2024-11-22T18:55:50,935 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-22T18:55:50,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:50,935 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2902): Flushing 0fc957929e5be1c291085a12bd8cf851 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-22T18:55:50,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/acac561e6dac4a949de4760b85a42e44 is 50, key is test_row_0/A:col10/1732301749638/Put/seqid=0 2024-11-22T18:55:50,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741935_1111 (size=9857) 2024-11-22T18:55:51,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-22T18:55:51,330 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/504de3f8256c4bcda61007c6106b9691 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/504de3f8256c4bcda61007c6106b9691 2024-11-22T18:55:51,330 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/cd1dd6b9f2164454bc49620caec46e6e as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/cd1dd6b9f2164454bc49620caec46e6e 2024-11-22T18:55:51,336 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0fc957929e5be1c291085a12bd8cf851/A of 0fc957929e5be1c291085a12bd8cf851 into 504de3f8256c4bcda61007c6106b9691(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:55:51,336 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0fc957929e5be1c291085a12bd8cf851/B of 0fc957929e5be1c291085a12bd8cf851 into cd1dd6b9f2164454bc49620caec46e6e(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:55:51,336 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fc957929e5be1c291085a12bd8cf851: 2024-11-22T18:55:51,336 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fc957929e5be1c291085a12bd8cf851: 2024-11-22T18:55:51,336 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., storeName=0fc957929e5be1c291085a12bd8cf851/A, priority=12, startTime=1732301750906; duration=0sec 2024-11-22T18:55:51,336 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., storeName=0fc957929e5be1c291085a12bd8cf851/B, priority=12, startTime=1732301750906; duration=0sec 2024-11-22T18:55:51,336 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:55:51,336 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fc957929e5be1c291085a12bd8cf851:A 2024-11-22T18:55:51,336 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:55:51,336 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fc957929e5be1c291085a12bd8cf851:B 2024-11-22T18:55:51,337 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T18:55:51,338 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47714 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T18:55:51,338 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1541): 0fc957929e5be1c291085a12bd8cf851/C is initiating minor compaction (all files) 2024-11-22T18:55:51,338 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fc957929e5be1c291085a12bd8cf851/C in TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:51,338 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/e0dc9a30462545ff9dd37b2cc6fcb802, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/79515d36d4ae4846896a0592c1b05cc0, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/eb9df5a862154ff381b809bc01521f43, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/68fb4e6e0a6b4c90863f63afd8da13c9] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp, totalSize=46.6 K 2024-11-22T18:55:51,338 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0dc9a30462545ff9dd37b2cc6fcb802, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1732301747838 2024-11-22T18:55:51,339 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79515d36d4ae4846896a0592c1b05cc0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1732301747981 2024-11-22T18:55:51,339 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb9df5a862154ff381b809bc01521f43, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1732301748117 2024-11-22T18:55:51,339 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68fb4e6e0a6b4c90863f63afd8da13c9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1732301748482 2024-11-22T18:55:51,343 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=487 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/acac561e6dac4a949de4760b85a42e44 2024-11-22T18:55:51,347 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fc957929e5be1c291085a12bd8cf851#C#compaction#99 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:55:51,348 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/32cee7a8efa14292ae35bca48b8688f4 is 50, key is test_row_0/C:col10/1732301749627/Put/seqid=0 2024-11-22T18:55:51,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/a60243bdf5aa458c95b36eb4e8318e7b is 50, key is test_row_0/B:col10/1732301749638/Put/seqid=0 2024-11-22T18:55:51,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741936_1112 (size=13391) 2024-11-22T18:55:51,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741937_1113 (size=9857) 2024-11-22T18:55:51,354 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=487 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/a60243bdf5aa458c95b36eb4e8318e7b 2024-11-22T18:55:51,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/1bd7e3f3ac8d4eb08e2d503c88116001 is 50, key is test_row_0/C:col10/1732301749638/Put/seqid=0 2024-11-22T18:55:51,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741938_1114 (size=9857) 2024-11-22T18:55:51,758 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/32cee7a8efa14292ae35bca48b8688f4 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/32cee7a8efa14292ae35bca48b8688f4 2024-11-22T18:55:51,763 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0fc957929e5be1c291085a12bd8cf851/C of 0fc957929e5be1c291085a12bd8cf851 into 32cee7a8efa14292ae35bca48b8688f4(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:55:51,763 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fc957929e5be1c291085a12bd8cf851: 2024-11-22T18:55:51,763 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., storeName=0fc957929e5be1c291085a12bd8cf851/C, priority=12, startTime=1732301750906; duration=0sec 2024-11-22T18:55:51,763 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:55:51,763 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fc957929e5be1c291085a12bd8cf851:C 2024-11-22T18:55:51,764 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=487 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/1bd7e3f3ac8d4eb08e2d503c88116001 2024-11-22T18:55:51,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/acac561e6dac4a949de4760b85a42e44 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/acac561e6dac4a949de4760b85a42e44 2024-11-22T18:55:51,772 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/acac561e6dac4a949de4760b85a42e44, entries=100, sequenceid=487, filesize=9.6 K 2024-11-22T18:55:51,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/a60243bdf5aa458c95b36eb4e8318e7b as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/a60243bdf5aa458c95b36eb4e8318e7b 2024-11-22T18:55:51,777 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/a60243bdf5aa458c95b36eb4e8318e7b, entries=100, sequenceid=487, filesize=9.6 K 2024-11-22T18:55:51,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/1bd7e3f3ac8d4eb08e2d503c88116001 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/1bd7e3f3ac8d4eb08e2d503c88116001 2024-11-22T18:55:51,781 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/1bd7e3f3ac8d4eb08e2d503c88116001, entries=100, sequenceid=487, filesize=9.6 K 2024-11-22T18:55:51,782 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 0fc957929e5be1c291085a12bd8cf851 in 847ms, sequenceid=487, compaction requested=false 2024-11-22T18:55:51,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2603): Flush status journal for 0fc957929e5be1c291085a12bd8cf851: 2024-11-22T18:55:51,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:51,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=30 2024-11-22T18:55:51,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster(4169): Remote procedure done, pid=30 2024-11-22T18:55:51,785 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-22T18:55:51,785 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6150 sec 2024-11-22T18:55:51,787 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-22T18:55:51,787 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-22T18:55:51,787 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T18:55:51,787 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T18:55:51,787 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:55:51,787 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:55:51,787 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:55:51,787 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:55:51,787 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T18:55:51,787 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T18:55:51,787 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees in 1.6200 sec 2024-11-22T18:55:51,796 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-22T18:55:51,796 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T18:55:51,796 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:55:51,797 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:55:51,797 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T18:55:52,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-22T18:55:52,316 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T18:55:52,428 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-22T18:55:57,578 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-22T18:55:57,578 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T18:55:57,578 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:55:57,578 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:55:57,578 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T18:55:57,597 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-22T18:55:57,597 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T18:55:57,597 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:55:57,597 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:55:57,597 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T18:55:57,597 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-22T18:55:57,597 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 108 2024-11-22T18:55:57,597 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 33 2024-11-22T18:55:57,597 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 107 2024-11-22T18:55:57,597 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 114 2024-11-22T18:55:57,597 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 30 2024-11-22T18:55:57,597 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-22T18:55:57,597 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2906 2024-11-22T18:55:57,597 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3004 2024-11-22T18:55:57,597 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-22T18:55:57,598 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1381 2024-11-22T18:55:57,598 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4133 rows 2024-11-22T18:55:57,598 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1368 2024-11-22T18:55:57,598 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4100 rows 2024-11-22T18:55:57,598 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-22T18:55:57,598 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T18:55:57,598 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:55:57,598 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:55:57,598 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T18:55:57,602 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-22T18:55:57,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-22T18:55:57,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-22T18:55:57,609 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732301757609"}]},"ts":"1732301757609"} 2024-11-22T18:55:57,611 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-22T18:55:57,611 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-22T18:55:57,612 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-22T18:55:57,616 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0fc957929e5be1c291085a12bd8cf851, UNASSIGN}] 2024-11-22T18:55:57,617 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0fc957929e5be1c291085a12bd8cf851, UNASSIGN 2024-11-22T18:55:57,618 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=0fc957929e5be1c291085a12bd8cf851, regionState=CLOSING, regionLocation=d79ba0c344fb,43593,1732301724173 2024-11-22T18:55:57,620 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0fc957929e5be1c291085a12bd8cf851, UNASSIGN because future has completed 2024-11-22T18:55:57,621 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-22T18:55:57,621 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173}] 2024-11-22T18:55:57,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-22T18:55:57,778 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(122): Close 0fc957929e5be1c291085a12bd8cf851 2024-11-22T18:55:57,778 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-22T18:55:57,779 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1722): Closing 0fc957929e5be1c291085a12bd8cf851, disabling compactions & flushes 2024-11-22T18:55:57,779 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:57,779 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:57,779 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. after waiting 0 ms 2024-11-22T18:55:57,779 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:57,779 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(2902): Flushing 0fc957929e5be1c291085a12bd8cf851 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-22T18:55:57,784 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/864f8ee787ff4d1aa599c24a376497af is 50, key is test_row_0/A:col10/1732301751785/Put/seqid=0 2024-11-22T18:55:57,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741939_1115 (size=12301) 2024-11-22T18:55:57,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-22T18:55:58,189 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=498 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/864f8ee787ff4d1aa599c24a376497af 2024-11-22T18:55:58,197 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/43fa5ac29c654767a97d080516615965 is 50, key is test_row_0/B:col10/1732301751785/Put/seqid=0 2024-11-22T18:55:58,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741940_1116 (size=12301) 2024-11-22T18:55:58,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-22T18:55:58,601 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=498 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/43fa5ac29c654767a97d080516615965 2024-11-22T18:55:58,608 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/8b7fc913fc9049eda3da79d2e07b295e is 50, key is test_row_0/C:col10/1732301751785/Put/seqid=0 2024-11-22T18:55:58,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741941_1117 (size=12301) 2024-11-22T18:55:58,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-22T18:55:59,013 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=498 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/8b7fc913fc9049eda3da79d2e07b295e 2024-11-22T18:55:59,018 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/A/864f8ee787ff4d1aa599c24a376497af as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/864f8ee787ff4d1aa599c24a376497af 2024-11-22T18:55:59,023 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/864f8ee787ff4d1aa599c24a376497af, entries=150, sequenceid=498, filesize=12.0 K 2024-11-22T18:55:59,024 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/B/43fa5ac29c654767a97d080516615965 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/43fa5ac29c654767a97d080516615965 2024-11-22T18:55:59,029 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/43fa5ac29c654767a97d080516615965, entries=150, sequenceid=498, filesize=12.0 K 2024-11-22T18:55:59,030 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/.tmp/C/8b7fc913fc9049eda3da79d2e07b295e as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/8b7fc913fc9049eda3da79d2e07b295e 2024-11-22T18:55:59,035 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/8b7fc913fc9049eda3da79d2e07b295e, entries=150, sequenceid=498, filesize=12.0 K 2024-11-22T18:55:59,036 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 0fc957929e5be1c291085a12bd8cf851 in 1257ms, sequenceid=498, compaction requested=true 2024-11-22T18:55:59,037 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/530a42fe465c4de3aa16c654d0b2d64b, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/1f4e77d17cbe4deead62ce1933cb5be1, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/170069664e754b08980f680460e8fdfb, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/68bc0ffc9d98419182f133302b83f612, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/df4993160e9c4b96a65479bb49bfd920, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/e75afde1f7d542139c31960498aeb306, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/c773ca82cb6d4c0d909aa064b2a0413d, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/5b59a42053de4c5aba20580e18de719a, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/dbf1f38b0c1c497ba1b9976ab9c80301, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/f1a3ac8e9d464b5a906d51bd26a808c8, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/76f5cc1dc97f41779e3ce0d0c0336ca3, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/6d7ea4d2e3f74246b4178825a7a7e23f, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/3c00670513f348198d438ff1e13a25d3, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/54773f3c769643429b718ecc29ef2e35, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/db7d5f72fa7e4227a89a1fadc8a5b21d, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/be9abcf3a5954079bc066fd563b998a9, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/3f3b4f29b00f471ab26d9d33aa8c6813, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/8e27507c5e0041d8afc6ca534cc7a150, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/e437ccc403d244f196ba86b1d456b3b0, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/9a303423e30946e18acb3554c9cbfd55, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/28442b558a6b499a9b1fb0494134c1b2, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/dda48163fbf0431b9a6a3d984a7b046c, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/c30c963951ec48da8dd7b04ffdb227a2, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/5e7d520937f94e0abdfea4ad43b2f941, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/faff56203fd949af900072b1549aa477, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/83e6fb16e6fe49d88a8fa38bacabc2d3, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/f4a7a6e0a8be41f79004bb940cf7c45a, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/56cd757edd5041bb8ed7ff66ae945d31, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/ae5e927aa37d4010b6b6fa0b888c5981, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/42cbbe96e24548f9b122096d0d2d64e3, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/44aae49019ab4da1be7a624deb925112, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/afd43bb1783741c6a2c6c28df29fc588] to archive 2024-11-22T18:55:59,040 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T18:55:59,046 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/530a42fe465c4de3aa16c654d0b2d64b to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/530a42fe465c4de3aa16c654d0b2d64b 2024-11-22T18:55:59,047 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/1f4e77d17cbe4deead62ce1933cb5be1 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/1f4e77d17cbe4deead62ce1933cb5be1 2024-11-22T18:55:59,048 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/170069664e754b08980f680460e8fdfb to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/170069664e754b08980f680460e8fdfb 2024-11-22T18:55:59,050 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/68bc0ffc9d98419182f133302b83f612 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/68bc0ffc9d98419182f133302b83f612 2024-11-22T18:55:59,051 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/df4993160e9c4b96a65479bb49bfd920 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/df4993160e9c4b96a65479bb49bfd920 2024-11-22T18:55:59,053 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/e75afde1f7d542139c31960498aeb306 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/e75afde1f7d542139c31960498aeb306 2024-11-22T18:55:59,054 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/c773ca82cb6d4c0d909aa064b2a0413d to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/c773ca82cb6d4c0d909aa064b2a0413d 2024-11-22T18:55:59,055 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/5b59a42053de4c5aba20580e18de719a to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/5b59a42053de4c5aba20580e18de719a 2024-11-22T18:55:59,056 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/dbf1f38b0c1c497ba1b9976ab9c80301 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/dbf1f38b0c1c497ba1b9976ab9c80301 2024-11-22T18:55:59,057 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/f1a3ac8e9d464b5a906d51bd26a808c8 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/f1a3ac8e9d464b5a906d51bd26a808c8 2024-11-22T18:55:59,058 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/76f5cc1dc97f41779e3ce0d0c0336ca3 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/76f5cc1dc97f41779e3ce0d0c0336ca3 2024-11-22T18:55:59,059 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/6d7ea4d2e3f74246b4178825a7a7e23f to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/6d7ea4d2e3f74246b4178825a7a7e23f 2024-11-22T18:55:59,060 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/3c00670513f348198d438ff1e13a25d3 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/3c00670513f348198d438ff1e13a25d3 2024-11-22T18:55:59,062 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/54773f3c769643429b718ecc29ef2e35 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/54773f3c769643429b718ecc29ef2e35 2024-11-22T18:55:59,063 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/db7d5f72fa7e4227a89a1fadc8a5b21d to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/db7d5f72fa7e4227a89a1fadc8a5b21d 2024-11-22T18:55:59,064 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/be9abcf3a5954079bc066fd563b998a9 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/be9abcf3a5954079bc066fd563b998a9 2024-11-22T18:55:59,066 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/3f3b4f29b00f471ab26d9d33aa8c6813 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/3f3b4f29b00f471ab26d9d33aa8c6813 2024-11-22T18:55:59,067 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/8e27507c5e0041d8afc6ca534cc7a150 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/8e27507c5e0041d8afc6ca534cc7a150 2024-11-22T18:55:59,068 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/e437ccc403d244f196ba86b1d456b3b0 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/e437ccc403d244f196ba86b1d456b3b0 2024-11-22T18:55:59,070 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/9a303423e30946e18acb3554c9cbfd55 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/9a303423e30946e18acb3554c9cbfd55 2024-11-22T18:55:59,072 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/28442b558a6b499a9b1fb0494134c1b2 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/28442b558a6b499a9b1fb0494134c1b2 2024-11-22T18:55:59,073 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/dda48163fbf0431b9a6a3d984a7b046c to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/dda48163fbf0431b9a6a3d984a7b046c 2024-11-22T18:55:59,074 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/c30c963951ec48da8dd7b04ffdb227a2 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/c30c963951ec48da8dd7b04ffdb227a2 2024-11-22T18:55:59,076 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/5e7d520937f94e0abdfea4ad43b2f941 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/5e7d520937f94e0abdfea4ad43b2f941 2024-11-22T18:55:59,078 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/faff56203fd949af900072b1549aa477 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/faff56203fd949af900072b1549aa477 2024-11-22T18:55:59,079 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/83e6fb16e6fe49d88a8fa38bacabc2d3 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/83e6fb16e6fe49d88a8fa38bacabc2d3 2024-11-22T18:55:59,080 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/f4a7a6e0a8be41f79004bb940cf7c45a to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/f4a7a6e0a8be41f79004bb940cf7c45a 2024-11-22T18:55:59,082 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/56cd757edd5041bb8ed7ff66ae945d31 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/56cd757edd5041bb8ed7ff66ae945d31 2024-11-22T18:55:59,084 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/ae5e927aa37d4010b6b6fa0b888c5981 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/ae5e927aa37d4010b6b6fa0b888c5981 2024-11-22T18:55:59,085 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/42cbbe96e24548f9b122096d0d2d64e3 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/42cbbe96e24548f9b122096d0d2d64e3 2024-11-22T18:55:59,086 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/44aae49019ab4da1be7a624deb925112 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/44aae49019ab4da1be7a624deb925112 2024-11-22T18:55:59,087 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/afd43bb1783741c6a2c6c28df29fc588 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/afd43bb1783741c6a2c6c28df29fc588 2024-11-22T18:55:59,101 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/6616ca09ab9e456c9f71da3c13334460, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/5304e9a1622746cd996429ba8355d625, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/167c457aadce411797b9242c311237b1, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/03902c96c6c544109a2f6077ea555546, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/30c3bcc79b0b4d079ea0087669192a67, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/e6b9a91556e0420b8ebc8739eb3c29e2, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/7d86e1721dc24b519cae934041cb5b8f, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/db5902d6b174425dbc93bde571672e15, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/cf719ea32ba44e1f8def5f11068443c0, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/a18bc7005e9c435ea64372bb15738d7d, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/7662f722c1874f70a8d83539e7029011, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/39730ab81f104679bb8bbf6f56b15c60, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/4cbf13b151bc4de9ae91481fe671a666, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/8c284efb46ad4e2eba4a3f1b43aa434f, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/58c3a4207100450ead6c81637717c59b, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/8c0885899fed4e0e95da32956ebe935c, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/f5a6861a41544d3a9132cd5bcf8ee301, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/5e4ddc9d6e11499b96376034bee60261, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/95ca925bd5034daab5f1dc781a263c3f, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/4e70d6ac9ea749919cc1c7ec08963987, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/13e6881ab4e74c0d8474e53164286249, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/8877606198044f09ab3554add83069c1, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/c1507559441d4888868fdebc3cda9591, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/421f01379dd843c880784f6d24d62a22, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/90e834a9d6df44cf88a4ce05b4781880, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/980677100bb34d83aa3fd8a34c8d6356, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/b382dea8320c4bf1b65d4e6459785d87, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/2863e5527ad8424fbab573bb31ee50d0, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/d3787c062ca1433095df13b8206ce4d4, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/820219ead312435f9b4c32193505849f, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/efc4e2d505fc4f009568798607ea9e03, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/93f7b603ffcb4352a1a015e85076f3a0] to archive 2024-11-22T18:55:59,102 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T18:55:59,104 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/6616ca09ab9e456c9f71da3c13334460 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/6616ca09ab9e456c9f71da3c13334460 2024-11-22T18:55:59,106 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/5304e9a1622746cd996429ba8355d625 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/5304e9a1622746cd996429ba8355d625 2024-11-22T18:55:59,107 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/167c457aadce411797b9242c311237b1 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/167c457aadce411797b9242c311237b1 2024-11-22T18:55:59,108 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/03902c96c6c544109a2f6077ea555546 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/03902c96c6c544109a2f6077ea555546 2024-11-22T18:55:59,110 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/30c3bcc79b0b4d079ea0087669192a67 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/30c3bcc79b0b4d079ea0087669192a67 2024-11-22T18:55:59,111 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/e6b9a91556e0420b8ebc8739eb3c29e2 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/e6b9a91556e0420b8ebc8739eb3c29e2 2024-11-22T18:55:59,112 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/7d86e1721dc24b519cae934041cb5b8f to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/7d86e1721dc24b519cae934041cb5b8f 2024-11-22T18:55:59,113 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/db5902d6b174425dbc93bde571672e15 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/db5902d6b174425dbc93bde571672e15 2024-11-22T18:55:59,115 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/cf719ea32ba44e1f8def5f11068443c0 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/cf719ea32ba44e1f8def5f11068443c0 2024-11-22T18:55:59,116 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/a18bc7005e9c435ea64372bb15738d7d to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/a18bc7005e9c435ea64372bb15738d7d 2024-11-22T18:55:59,117 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/7662f722c1874f70a8d83539e7029011 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/7662f722c1874f70a8d83539e7029011 2024-11-22T18:55:59,119 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/39730ab81f104679bb8bbf6f56b15c60 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/39730ab81f104679bb8bbf6f56b15c60 2024-11-22T18:55:59,120 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/4cbf13b151bc4de9ae91481fe671a666 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/4cbf13b151bc4de9ae91481fe671a666 2024-11-22T18:55:59,121 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/8c284efb46ad4e2eba4a3f1b43aa434f to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/8c284efb46ad4e2eba4a3f1b43aa434f 2024-11-22T18:55:59,122 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/58c3a4207100450ead6c81637717c59b to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/58c3a4207100450ead6c81637717c59b 2024-11-22T18:55:59,124 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/8c0885899fed4e0e95da32956ebe935c to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/8c0885899fed4e0e95da32956ebe935c 2024-11-22T18:55:59,125 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/f5a6861a41544d3a9132cd5bcf8ee301 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/f5a6861a41544d3a9132cd5bcf8ee301 2024-11-22T18:55:59,126 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/5e4ddc9d6e11499b96376034bee60261 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/5e4ddc9d6e11499b96376034bee60261 2024-11-22T18:55:59,127 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/95ca925bd5034daab5f1dc781a263c3f to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/95ca925bd5034daab5f1dc781a263c3f 2024-11-22T18:55:59,128 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/4e70d6ac9ea749919cc1c7ec08963987 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/4e70d6ac9ea749919cc1c7ec08963987 2024-11-22T18:55:59,130 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/13e6881ab4e74c0d8474e53164286249 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/13e6881ab4e74c0d8474e53164286249 2024-11-22T18:55:59,131 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/8877606198044f09ab3554add83069c1 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/8877606198044f09ab3554add83069c1 2024-11-22T18:55:59,132 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/c1507559441d4888868fdebc3cda9591 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/c1507559441d4888868fdebc3cda9591 2024-11-22T18:55:59,133 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/421f01379dd843c880784f6d24d62a22 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/421f01379dd843c880784f6d24d62a22 2024-11-22T18:55:59,134 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/90e834a9d6df44cf88a4ce05b4781880 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/90e834a9d6df44cf88a4ce05b4781880 2024-11-22T18:55:59,135 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/980677100bb34d83aa3fd8a34c8d6356 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/980677100bb34d83aa3fd8a34c8d6356 2024-11-22T18:55:59,137 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/b382dea8320c4bf1b65d4e6459785d87 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/b382dea8320c4bf1b65d4e6459785d87 2024-11-22T18:55:59,138 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/2863e5527ad8424fbab573bb31ee50d0 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/2863e5527ad8424fbab573bb31ee50d0 2024-11-22T18:55:59,139 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/d3787c062ca1433095df13b8206ce4d4 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/d3787c062ca1433095df13b8206ce4d4 2024-11-22T18:55:59,140 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/820219ead312435f9b4c32193505849f to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/820219ead312435f9b4c32193505849f 2024-11-22T18:55:59,142 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/efc4e2d505fc4f009568798607ea9e03 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/efc4e2d505fc4f009568798607ea9e03 2024-11-22T18:55:59,143 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/93f7b603ffcb4352a1a015e85076f3a0 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/93f7b603ffcb4352a1a015e85076f3a0 2024-11-22T18:55:59,144 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/c45836aee3c643dfa7a9f228a0926a2b, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/ffbe2484e7754ff089bb9b8da23c538d, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/f1bd7a68a09b48cd9ad291aa308f0689, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/9c71c5ce5d5f4bdbabb6d078ac2814c6, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/1946508858304f98b3f42766981dc77b, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/71e663aa1a024fe09b66c0985ded942c, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/0528648d1a5544fb8b66cdc77460ed48, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/242ffc8934484b99800b8a0ab9c672cb, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/0973836890f045978d0d7961f119fb26, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/c3907bd32f484139a2611b1de63714e4, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/e7de47761f2846ed850d07bdb274fea6, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/dd50a222a7834a05bcb2e20808f050c2, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/7d52149821a74922ae542acd73064b43, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/bc85ad73421f42d4b07a6e81ab9c4fce, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/ca5be2fb67274703a27290081a2748ff, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/611d036604e2408ba125a29638a09507, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/6c7a55a6a84243899b33b3da989028e7, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/b16cc56652e14844a6a1f6746233f9fa, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/345fccfb3b8c4a84ba8987d15d9f809a, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/321653053576407eb3fec52a2824f743, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/33bd6f57c516412ca560f91b7bced824, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/421945da95c24d9fa32f895900b2c4f9, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/dab2debff09746439889655f2f8cca0b, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/5918205efb034fc886836d11e301b29b, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/49ae493c536e49298fc2faa270949f9e, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/f3303a5db0f34ef588e9b2968691210a, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/c9db5bebfce640a9a3aa72c445678da2, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/e0dc9a30462545ff9dd37b2cc6fcb802, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/2b298136f05e409d96e3c937680223db, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/79515d36d4ae4846896a0592c1b05cc0, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/eb9df5a862154ff381b809bc01521f43, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/68fb4e6e0a6b4c90863f63afd8da13c9] to archive 2024-11-22T18:55:59,145 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T18:55:59,147 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/c45836aee3c643dfa7a9f228a0926a2b to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/c45836aee3c643dfa7a9f228a0926a2b 2024-11-22T18:55:59,148 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/ffbe2484e7754ff089bb9b8da23c538d to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/ffbe2484e7754ff089bb9b8da23c538d 2024-11-22T18:55:59,150 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/f1bd7a68a09b48cd9ad291aa308f0689 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/f1bd7a68a09b48cd9ad291aa308f0689 2024-11-22T18:55:59,151 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/9c71c5ce5d5f4bdbabb6d078ac2814c6 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/9c71c5ce5d5f4bdbabb6d078ac2814c6 2024-11-22T18:55:59,152 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/1946508858304f98b3f42766981dc77b to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/1946508858304f98b3f42766981dc77b 2024-11-22T18:55:59,153 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/71e663aa1a024fe09b66c0985ded942c to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/71e663aa1a024fe09b66c0985ded942c 2024-11-22T18:55:59,155 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/0528648d1a5544fb8b66cdc77460ed48 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/0528648d1a5544fb8b66cdc77460ed48 2024-11-22T18:55:59,156 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/242ffc8934484b99800b8a0ab9c672cb to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/242ffc8934484b99800b8a0ab9c672cb 2024-11-22T18:55:59,157 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/0973836890f045978d0d7961f119fb26 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/0973836890f045978d0d7961f119fb26 2024-11-22T18:55:59,158 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/c3907bd32f484139a2611b1de63714e4 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/c3907bd32f484139a2611b1de63714e4 2024-11-22T18:55:59,159 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/e7de47761f2846ed850d07bdb274fea6 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/e7de47761f2846ed850d07bdb274fea6 2024-11-22T18:55:59,160 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/dd50a222a7834a05bcb2e20808f050c2 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/dd50a222a7834a05bcb2e20808f050c2 2024-11-22T18:55:59,162 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/7d52149821a74922ae542acd73064b43 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/7d52149821a74922ae542acd73064b43 2024-11-22T18:55:59,163 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/bc85ad73421f42d4b07a6e81ab9c4fce to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/bc85ad73421f42d4b07a6e81ab9c4fce 2024-11-22T18:55:59,164 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/ca5be2fb67274703a27290081a2748ff to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/ca5be2fb67274703a27290081a2748ff 2024-11-22T18:55:59,165 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/611d036604e2408ba125a29638a09507 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/611d036604e2408ba125a29638a09507 2024-11-22T18:55:59,166 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/6c7a55a6a84243899b33b3da989028e7 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/6c7a55a6a84243899b33b3da989028e7 2024-11-22T18:55:59,167 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/b16cc56652e14844a6a1f6746233f9fa to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/b16cc56652e14844a6a1f6746233f9fa 2024-11-22T18:55:59,169 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/345fccfb3b8c4a84ba8987d15d9f809a to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/345fccfb3b8c4a84ba8987d15d9f809a 2024-11-22T18:55:59,170 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/321653053576407eb3fec52a2824f743 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/321653053576407eb3fec52a2824f743 2024-11-22T18:55:59,172 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/33bd6f57c516412ca560f91b7bced824 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/33bd6f57c516412ca560f91b7bced824 2024-11-22T18:55:59,173 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/421945da95c24d9fa32f895900b2c4f9 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/421945da95c24d9fa32f895900b2c4f9 2024-11-22T18:55:59,174 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/dab2debff09746439889655f2f8cca0b to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/dab2debff09746439889655f2f8cca0b 2024-11-22T18:55:59,175 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/5918205efb034fc886836d11e301b29b to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/5918205efb034fc886836d11e301b29b 2024-11-22T18:55:59,176 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/49ae493c536e49298fc2faa270949f9e to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/49ae493c536e49298fc2faa270949f9e 2024-11-22T18:55:59,177 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/f3303a5db0f34ef588e9b2968691210a to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/f3303a5db0f34ef588e9b2968691210a 2024-11-22T18:55:59,179 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/c9db5bebfce640a9a3aa72c445678da2 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/c9db5bebfce640a9a3aa72c445678da2 2024-11-22T18:55:59,180 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/e0dc9a30462545ff9dd37b2cc6fcb802 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/e0dc9a30462545ff9dd37b2cc6fcb802 2024-11-22T18:55:59,181 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/2b298136f05e409d96e3c937680223db to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/2b298136f05e409d96e3c937680223db 2024-11-22T18:55:59,183 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/79515d36d4ae4846896a0592c1b05cc0 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/79515d36d4ae4846896a0592c1b05cc0 2024-11-22T18:55:59,184 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/eb9df5a862154ff381b809bc01521f43 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/eb9df5a862154ff381b809bc01521f43 2024-11-22T18:55:59,185 DEBUG [StoreCloser-TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/68fb4e6e0a6b4c90863f63afd8da13c9 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/68fb4e6e0a6b4c90863f63afd8da13c9 2024-11-22T18:55:59,190 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/recovered.edits/501.seqid, newMaxSeqId=501, maxSeqId=1 2024-11-22T18:55:59,191 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. 2024-11-22T18:55:59,191 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1676): Region close journal for 0fc957929e5be1c291085a12bd8cf851: Waiting for close lock at 1732301757779Running coprocessor pre-close hooks at 1732301757779Disabling compacts and flushes for region at 1732301757779Disabling writes for close at 1732301757779Obtaining lock to block concurrent updates at 1732301757779Preparing flush snapshotting stores in 0fc957929e5be1c291085a12bd8cf851 at 1732301757779Finished memstore snapshotting TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1732301757780 (+1 ms)Flushing stores of TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851. at 1732301757780Flushing 0fc957929e5be1c291085a12bd8cf851/A: creating writer at 1732301757780Flushing 0fc957929e5be1c291085a12bd8cf851/A: appending metadata at 1732301757783 (+3 ms)Flushing 0fc957929e5be1c291085a12bd8cf851/A: closing flushed file at 1732301757783Flushing 0fc957929e5be1c291085a12bd8cf851/B: creating writer at 1732301758193 (+410 ms)Flushing 0fc957929e5be1c291085a12bd8cf851/B: appending metadata at 1732301758196 (+3 ms)Flushing 0fc957929e5be1c291085a12bd8cf851/B: closing flushed file at 1732301758196Flushing 0fc957929e5be1c291085a12bd8cf851/C: creating writer at 1732301758605 (+409 ms)Flushing 0fc957929e5be1c291085a12bd8cf851/C: appending metadata at 1732301758608 (+3 ms)Flushing 0fc957929e5be1c291085a12bd8cf851/C: closing flushed file at 1732301758608Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1bb74573: reopening flushed file at 1732301759017 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@110c2ff4: reopening flushed file at 1732301759023 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1858fcdd: reopening flushed file at 1732301759029 (+6 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 0fc957929e5be1c291085a12bd8cf851 in 1257ms, sequenceid=498, compaction requested=true at 1732301759036 (+7 ms)Writing region close event to WAL at 1732301759187 (+151 ms)Running coprocessor post-close hooks at 1732301759191 (+4 ms)Closed at 1732301759191 2024-11-22T18:55:59,194 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(157): Closed 0fc957929e5be1c291085a12bd8cf851 2024-11-22T18:55:59,195 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=0fc957929e5be1c291085a12bd8cf851, regionState=CLOSED 2024-11-22T18:55:59,196 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 because future has completed 2024-11-22T18:55:59,199 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-22T18:55:59,199 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 0fc957929e5be1c291085a12bd8cf851, server=d79ba0c344fb,43593,1732301724173 in 1.5760 sec 2024-11-22T18:55:59,201 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-22T18:55:59,201 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0fc957929e5be1c291085a12bd8cf851, UNASSIGN in 1.5830 sec 2024-11-22T18:55:59,203 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-22T18:55:59,203 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5900 sec 2024-11-22T18:55:59,205 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732301759205"}]},"ts":"1732301759205"} 2024-11-22T18:55:59,207 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-22T18:55:59,207 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-22T18:55:59,209 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6030 sec 2024-11-22T18:55:59,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-22T18:55:59,756 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-22T18:55:59,759 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-22T18:55:59,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] procedure2.ProcedureExecutor(1139): Stored pid=35, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T18:55:59,765 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=35, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T18:55:59,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-11-22T18:55:59,766 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=35, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T18:55:59,770 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851 2024-11-22T18:55:59,774 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A, FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B, FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C, FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/recovered.edits] 2024-11-22T18:55:59,777 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/504de3f8256c4bcda61007c6106b9691 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/504de3f8256c4bcda61007c6106b9691 2024-11-22T18:55:59,778 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/864f8ee787ff4d1aa599c24a376497af to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/864f8ee787ff4d1aa599c24a376497af 2024-11-22T18:55:59,779 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/acac561e6dac4a949de4760b85a42e44 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/A/acac561e6dac4a949de4760b85a42e44 2024-11-22T18:55:59,782 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/43fa5ac29c654767a97d080516615965 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/43fa5ac29c654767a97d080516615965 2024-11-22T18:55:59,783 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/a60243bdf5aa458c95b36eb4e8318e7b to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/a60243bdf5aa458c95b36eb4e8318e7b 2024-11-22T18:55:59,784 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/cd1dd6b9f2164454bc49620caec46e6e to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/B/cd1dd6b9f2164454bc49620caec46e6e 2024-11-22T18:55:59,786 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/1bd7e3f3ac8d4eb08e2d503c88116001 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/1bd7e3f3ac8d4eb08e2d503c88116001 2024-11-22T18:55:59,787 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/32cee7a8efa14292ae35bca48b8688f4 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/32cee7a8efa14292ae35bca48b8688f4 2024-11-22T18:55:59,788 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/8b7fc913fc9049eda3da79d2e07b295e to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/C/8b7fc913fc9049eda3da79d2e07b295e 2024-11-22T18:55:59,791 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/recovered.edits/501.seqid to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851/recovered.edits/501.seqid 2024-11-22T18:55:59,792 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/0fc957929e5be1c291085a12bd8cf851 2024-11-22T18:55:59,792 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-22T18:55:59,797 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=35, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T18:55:59,801 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-22T18:55:59,805 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-22T18:55:59,835 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-22T18:55:59,836 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=35, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T18:55:59,837 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-22T18:55:59,837 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732301759837"}]},"ts":"9223372036854775807"} 2024-11-22T18:55:59,841 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-22T18:55:59,841 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 0fc957929e5be1c291085a12bd8cf851, NAME => 'TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851.', STARTKEY => '', ENDKEY => ''}] 2024-11-22T18:55:59,841 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-22T18:55:59,841 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732301759841"}]},"ts":"9223372036854775807"} 2024-11-22T18:55:59,844 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-22T18:55:59,845 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=35, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T18:55:59,846 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 85 msec 2024-11-22T18:55:59,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-11-22T18:55:59,877 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-22T18:55:59,877 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-22T18:55:59,886 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=237 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;d79ba0c344fb:43593-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-258122325_22 at /127.0.0.1:40326 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=451 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=313 (was 265) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8024 (was 7688) - AvailableMemoryMB LEAK? - 2024-11-22T18:55:59,895 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=237, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=313, ProcessCount=11, AvailableMemoryMB=8024 2024-11-22T18:55:59,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-22T18:55:59,897 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T18:55:59,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-22T18:55:59,899 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-22T18:55:59,899 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:55:59,899 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 36 2024-11-22T18:55:59,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-22T18:55:59,900 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-22T18:55:59,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741942_1118 (size=959) 2024-11-22T18:56:00,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-22T18:56:00,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-22T18:56:00,308 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875 2024-11-22T18:56:00,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741943_1119 (size=53) 2024-11-22T18:56:00,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-22T18:56:00,714 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T18:56:00,714 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing e2eea2cf5b56baf2f173c11708a11448, disabling compactions & flushes 2024-11-22T18:56:00,714 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:00,714 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:00,714 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. after waiting 0 ms 2024-11-22T18:56:00,714 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:00,715 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:00,715 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for e2eea2cf5b56baf2f173c11708a11448: Waiting for close lock at 1732301760714Disabling compacts and flushes for region at 1732301760714Disabling writes for close at 1732301760714Writing region close event to WAL at 1732301760714Closed at 1732301760714 2024-11-22T18:56:00,716 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-22T18:56:00,716 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732301760716"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732301760716"}]},"ts":"1732301760716"} 2024-11-22T18:56:00,718 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-22T18:56:00,719 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-22T18:56:00,719 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732301760719"}]},"ts":"1732301760719"} 2024-11-22T18:56:00,720 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-22T18:56:00,720 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e2eea2cf5b56baf2f173c11708a11448, ASSIGN}] 2024-11-22T18:56:00,721 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e2eea2cf5b56baf2f173c11708a11448, ASSIGN 2024-11-22T18:56:00,722 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e2eea2cf5b56baf2f173c11708a11448, ASSIGN; state=OFFLINE, location=d79ba0c344fb,43593,1732301724173; forceNewPlan=false, retain=false 2024-11-22T18:56:00,873 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=e2eea2cf5b56baf2f173c11708a11448, regionState=OPENING, regionLocation=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:00,875 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e2eea2cf5b56baf2f173c11708a11448, ASSIGN because future has completed 2024-11-22T18:56:00,875 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173}] 2024-11-22T18:56:01,030 INFO [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:01,031 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7752): Opening region: {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} 2024-11-22T18:56:01,031 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:01,031 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T18:56:01,031 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7794): checking encryption for e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:01,031 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7797): checking classloading for e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:01,032 INFO [StoreOpener-e2eea2cf5b56baf2f173c11708a11448-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:01,035 INFO [StoreOpener-e2eea2cf5b56baf2f173c11708a11448-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e2eea2cf5b56baf2f173c11708a11448 columnFamilyName A 2024-11-22T18:56:01,035 DEBUG [StoreOpener-e2eea2cf5b56baf2f173c11708a11448-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:56:01,035 INFO [StoreOpener-e2eea2cf5b56baf2f173c11708a11448-1 {}] regionserver.HStore(327): Store=e2eea2cf5b56baf2f173c11708a11448/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T18:56:01,035 INFO [StoreOpener-e2eea2cf5b56baf2f173c11708a11448-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:01,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-22T18:56:01,037 INFO [StoreOpener-e2eea2cf5b56baf2f173c11708a11448-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e2eea2cf5b56baf2f173c11708a11448 columnFamilyName B 2024-11-22T18:56:01,037 DEBUG [StoreOpener-e2eea2cf5b56baf2f173c11708a11448-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:56:01,037 INFO [StoreOpener-e2eea2cf5b56baf2f173c11708a11448-1 {}] regionserver.HStore(327): Store=e2eea2cf5b56baf2f173c11708a11448/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T18:56:01,037 INFO [StoreOpener-e2eea2cf5b56baf2f173c11708a11448-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:01,038 INFO [StoreOpener-e2eea2cf5b56baf2f173c11708a11448-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e2eea2cf5b56baf2f173c11708a11448 columnFamilyName C 2024-11-22T18:56:01,038 DEBUG [StoreOpener-e2eea2cf5b56baf2f173c11708a11448-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:56:01,039 INFO [StoreOpener-e2eea2cf5b56baf2f173c11708a11448-1 {}] regionserver.HStore(327): Store=e2eea2cf5b56baf2f173c11708a11448/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T18:56:01,039 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1038): replaying wal for e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:01,039 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:01,040 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:01,041 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1048): stopping wal replay for e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:01,041 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1060): Cleaning up temporary data for e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:01,041 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-22T18:56:01,042 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): writing seq id for e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:01,044 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T18:56:01,044 INFO [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1114): Opened e2eea2cf5b56baf2f173c11708a11448; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68877963, jitterRate=0.026361629366874695}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-22T18:56:01,045 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:01,045 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1006): Region open journal for e2eea2cf5b56baf2f173c11708a11448: Running coprocessor pre-open hook at 1732301761031Writing region info on filesystem at 1732301761031Initializing all the Stores at 1732301761032 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732301761032Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732301761032Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732301761032Cleaning up temporary data from old regions at 1732301761041 (+9 ms)Running coprocessor post-open hooks at 1732301761045 (+4 ms)Region opened successfully at 1732301761045 2024-11-22T18:56:01,046 INFO [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., pid=38, masterSystemTime=1732301761027 2024-11-22T18:56:01,048 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:01,048 INFO [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:01,048 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=e2eea2cf5b56baf2f173c11708a11448, regionState=OPEN, openSeqNum=2, regionLocation=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:01,050 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 because future has completed 2024-11-22T18:56:01,053 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-22T18:56:01,054 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; OpenRegionProcedure e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 in 176 msec 2024-11-22T18:56:01,056 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-22T18:56:01,056 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e2eea2cf5b56baf2f173c11708a11448, ASSIGN in 334 msec 2024-11-22T18:56:01,057 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-22T18:56:01,057 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732301761057"}]},"ts":"1732301761057"} 2024-11-22T18:56:01,059 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-22T18:56:01,060 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-22T18:56:01,061 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1630 sec 2024-11-22T18:56:02,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-22T18:56:02,046 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-22T18:56:02,047 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ffee9aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:56:02,047 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d79ba0c344fb,44109,-1 for getting cluster id 2024-11-22T18:56:02,047 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T18:56:02,049 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '058c0dbe-7061-419a-804d-605a1c1b6341' 2024-11-22T18:56:02,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T18:56:02,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "058c0dbe-7061-419a-804d-605a1c1b6341" 2024-11-22T18:56:02,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4810677, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:56:02,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d79ba0c344fb,44109,-1] 2024-11-22T18:56:02,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T18:56:02,050 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:56:02,051 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51066, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T18:56:02,052 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76cf8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:56:02,052 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T18:56:02,053 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d79ba0c344fb,43593,1732301724173, seqNum=-1] 2024-11-22T18:56:02,053 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T18:56:02,054 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43424, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T18:56:02,056 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-22T18:56:02,057 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is d79ba0c344fb,44109,1732301723398 2024-11-22T18:56:02,057 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7e8f617 2024-11-22T18:56:02,057 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-22T18:56:02,058 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51082, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-22T18:56:02,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-22T18:56:02,063 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T18:56:02,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] procedure2.ProcedureExecutor(1139): Stored pid=39, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-22T18:56:02,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741944_1120 (size=995) 2024-11-22T18:56:02,482 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-22T18:56:02,482 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-22T18:56:02,486 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-22T18:56:02,492 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e2eea2cf5b56baf2f173c11708a11448, REOPEN/MOVE}] 2024-11-22T18:56:02,492 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e2eea2cf5b56baf2f173c11708a11448, REOPEN/MOVE 2024-11-22T18:56:02,493 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=41 updating hbase:meta row=e2eea2cf5b56baf2f173c11708a11448, regionState=CLOSING, regionLocation=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:02,495 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e2eea2cf5b56baf2f173c11708a11448, REOPEN/MOVE because future has completed 2024-11-22T18:56:02,495 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-22T18:56:02,495 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=41, state=RUNNABLE, hasLock=false; CloseRegionProcedure e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173}] 2024-11-22T18:56:02,648 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] handler.UnassignRegionHandler(122): Close e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:02,648 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-22T18:56:02,648 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1722): Closing e2eea2cf5b56baf2f173c11708a11448, disabling compactions & flushes 2024-11-22T18:56:02,648 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:02,648 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:02,648 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. after waiting 0 ms 2024-11-22T18:56:02,648 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:02,652 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-22T18:56:02,653 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:02,653 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1676): Region close journal for e2eea2cf5b56baf2f173c11708a11448: Waiting for close lock at 1732301762648Running coprocessor pre-close hooks at 1732301762648Disabling compacts and flushes for region at 1732301762648Disabling writes for close at 1732301762648Writing region close event to WAL at 1732301762649 (+1 ms)Running coprocessor post-close hooks at 1732301762653 (+4 ms)Closed at 1732301762653 2024-11-22T18:56:02,653 WARN [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegionServer(3299): Not adding moved region record: e2eea2cf5b56baf2f173c11708a11448 to self. 2024-11-22T18:56:02,655 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] handler.UnassignRegionHandler(157): Closed e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:02,656 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=41 updating hbase:meta row=e2eea2cf5b56baf2f173c11708a11448, regionState=CLOSED 2024-11-22T18:56:02,657 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=42, ppid=41, state=RUNNABLE, hasLock=false; CloseRegionProcedure e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 because future has completed 2024-11-22T18:56:02,659 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=41 2024-11-22T18:56:02,659 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=41, state=SUCCESS, hasLock=false; CloseRegionProcedure e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 in 163 msec 2024-11-22T18:56:02,660 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e2eea2cf5b56baf2f173c11708a11448, REOPEN/MOVE; state=CLOSED, location=d79ba0c344fb,43593,1732301724173; forceNewPlan=false, retain=true 2024-11-22T18:56:02,811 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=41 updating hbase:meta row=e2eea2cf5b56baf2f173c11708a11448, regionState=OPENING, regionLocation=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:02,813 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e2eea2cf5b56baf2f173c11708a11448, REOPEN/MOVE because future has completed 2024-11-22T18:56:02,813 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=41, state=RUNNABLE, hasLock=false; OpenRegionProcedure e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173}] 2024-11-22T18:56:02,968 INFO [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:02,968 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7752): Opening region: {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} 2024-11-22T18:56:02,969 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:02,969 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T18:56:02,969 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7794): checking encryption for e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:02,969 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7797): checking classloading for e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:02,973 INFO [StoreOpener-e2eea2cf5b56baf2f173c11708a11448-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:02,979 INFO [StoreOpener-e2eea2cf5b56baf2f173c11708a11448-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e2eea2cf5b56baf2f173c11708a11448 columnFamilyName A 2024-11-22T18:56:02,980 DEBUG [StoreOpener-e2eea2cf5b56baf2f173c11708a11448-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:56:02,981 INFO [StoreOpener-e2eea2cf5b56baf2f173c11708a11448-1 {}] regionserver.HStore(327): Store=e2eea2cf5b56baf2f173c11708a11448/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T18:56:02,981 INFO [StoreOpener-e2eea2cf5b56baf2f173c11708a11448-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:02,982 INFO [StoreOpener-e2eea2cf5b56baf2f173c11708a11448-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e2eea2cf5b56baf2f173c11708a11448 columnFamilyName B 2024-11-22T18:56:02,982 DEBUG [StoreOpener-e2eea2cf5b56baf2f173c11708a11448-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:56:02,982 INFO [StoreOpener-e2eea2cf5b56baf2f173c11708a11448-1 {}] regionserver.HStore(327): Store=e2eea2cf5b56baf2f173c11708a11448/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T18:56:02,982 INFO [StoreOpener-e2eea2cf5b56baf2f173c11708a11448-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:02,983 INFO [StoreOpener-e2eea2cf5b56baf2f173c11708a11448-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e2eea2cf5b56baf2f173c11708a11448 columnFamilyName C 2024-11-22T18:56:02,983 DEBUG [StoreOpener-e2eea2cf5b56baf2f173c11708a11448-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:56:02,983 INFO [StoreOpener-e2eea2cf5b56baf2f173c11708a11448-1 {}] regionserver.HStore(327): Store=e2eea2cf5b56baf2f173c11708a11448/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T18:56:02,983 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1038): replaying wal for e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:02,984 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:02,985 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:02,986 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1048): stopping wal replay for e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:02,986 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1060): Cleaning up temporary data for e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:02,986 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-22T18:56:02,987 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): writing seq id for e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:02,988 INFO [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1114): Opened e2eea2cf5b56baf2f173c11708a11448; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70462437, jitterRate=0.049972131848335266}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-22T18:56:02,988 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:02,990 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1006): Region open journal for e2eea2cf5b56baf2f173c11708a11448: Running coprocessor pre-open hook at 1732301762969Writing region info on filesystem at 1732301762969Initializing all the Stores at 1732301762970 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732301762970Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732301762970Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732301762970Cleaning up temporary data from old regions at 1732301762986 (+16 ms)Running coprocessor post-open hooks at 1732301762988 (+2 ms)Region opened successfully at 1732301762990 (+2 ms) 2024-11-22T18:56:02,991 INFO [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., pid=43, masterSystemTime=1732301762965 2024-11-22T18:56:02,993 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:02,993 INFO [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:02,994 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=41 updating hbase:meta row=e2eea2cf5b56baf2f173c11708a11448, regionState=OPEN, openSeqNum=5, regionLocation=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:02,996 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=41, state=RUNNABLE, hasLock=false; OpenRegionProcedure e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 because future has completed 2024-11-22T18:56:02,999 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=41 2024-11-22T18:56:02,999 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=41, state=SUCCESS, hasLock=false; OpenRegionProcedure e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 in 183 msec 2024-11-22T18:56:03,001 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-22T18:56:03,001 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e2eea2cf5b56baf2f173c11708a11448, REOPEN/MOVE in 507 msec 2024-11-22T18:56:03,005 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=39 2024-11-22T18:56:03,005 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=39, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 518 msec 2024-11-22T18:56:03,008 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 942 msec 2024-11-22T18:56:03,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-22T18:56:03,010 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-22T18:56:03,011 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f683619, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:56:03,011 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d79ba0c344fb,44109,-1 for getting cluster id 2024-11-22T18:56:03,011 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T18:56:03,012 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '058c0dbe-7061-419a-804d-605a1c1b6341' 2024-11-22T18:56:03,012 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T18:56:03,012 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "058c0dbe-7061-419a-804d-605a1c1b6341" 2024-11-22T18:56:03,012 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@270e099d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:56:03,012 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d79ba0c344fb,44109,-1] 2024-11-22T18:56:03,013 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T18:56:03,013 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:56:03,014 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51102, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T18:56:03,014 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@251113c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:56:03,015 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dd47c94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:56:03,015 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d79ba0c344fb,44109,-1 for getting cluster id 2024-11-22T18:56:03,015 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T18:56:03,016 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '058c0dbe-7061-419a-804d-605a1c1b6341' 2024-11-22T18:56:03,016 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T18:56:03,016 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "058c0dbe-7061-419a-804d-605a1c1b6341" 2024-11-22T18:56:03,016 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62d1f2da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:56:03,016 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d79ba0c344fb,44109,-1] 2024-11-22T18:56:03,017 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T18:56:03,017 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:56:03,017 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51128, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T18:56:03,018 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43166dce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:56:03,019 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@750c1845, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:56:03,019 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d79ba0c344fb,44109,-1 for getting cluster id 2024-11-22T18:56:03,019 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T18:56:03,020 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '058c0dbe-7061-419a-804d-605a1c1b6341' 2024-11-22T18:56:03,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T18:56:03,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "058c0dbe-7061-419a-804d-605a1c1b6341" 2024-11-22T18:56:03,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b571b86, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:56:03,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d79ba0c344fb,44109,-1] 2024-11-22T18:56:03,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T18:56:03,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:56:03,021 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51146, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T18:56:03,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52470120, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:56:03,022 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@85a5f7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:56:03,023 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d79ba0c344fb,44109,-1 for getting cluster id 2024-11-22T18:56:03,023 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T18:56:03,023 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '058c0dbe-7061-419a-804d-605a1c1b6341' 2024-11-22T18:56:03,023 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T18:56:03,024 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "058c0dbe-7061-419a-804d-605a1c1b6341" 2024-11-22T18:56:03,024 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35eea0f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:56:03,024 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d79ba0c344fb,44109,-1] 2024-11-22T18:56:03,024 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T18:56:03,024 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:56:03,025 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51168, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T18:56:03,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33f307cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:56:03,026 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71151d6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:56:03,026 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d79ba0c344fb,44109,-1 for getting cluster id 2024-11-22T18:56:03,026 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T18:56:03,027 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '058c0dbe-7061-419a-804d-605a1c1b6341' 2024-11-22T18:56:03,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T18:56:03,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "058c0dbe-7061-419a-804d-605a1c1b6341" 2024-11-22T18:56:03,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a5c699a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:56:03,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d79ba0c344fb,44109,-1] 2024-11-22T18:56:03,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T18:56:03,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:56:03,028 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51184, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T18:56:03,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f0db91d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:56:03,029 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ab1fb6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:56:03,029 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d79ba0c344fb,44109,-1 for getting cluster id 2024-11-22T18:56:03,029 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T18:56:03,030 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '058c0dbe-7061-419a-804d-605a1c1b6341' 2024-11-22T18:56:03,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T18:56:03,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "058c0dbe-7061-419a-804d-605a1c1b6341" 2024-11-22T18:56:03,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@671a0512, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:56:03,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d79ba0c344fb,44109,-1] 2024-11-22T18:56:03,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T18:56:03,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:56:03,032 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51200, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T18:56:03,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dd7435b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:56:03,033 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dd18334, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:56:03,033 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d79ba0c344fb,44109,-1 for getting cluster id 2024-11-22T18:56:03,033 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T18:56:03,033 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '058c0dbe-7061-419a-804d-605a1c1b6341' 2024-11-22T18:56:03,033 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T18:56:03,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "058c0dbe-7061-419a-804d-605a1c1b6341" 2024-11-22T18:56:03,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ac1445e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:56:03,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d79ba0c344fb,44109,-1] 2024-11-22T18:56:03,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T18:56:03,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:56:03,035 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51214, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T18:56:03,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@230fe799, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:56:03,036 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52c57d01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:56:03,036 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d79ba0c344fb,44109,-1 for getting cluster id 2024-11-22T18:56:03,036 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T18:56:03,036 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '058c0dbe-7061-419a-804d-605a1c1b6341' 2024-11-22T18:56:03,036 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T18:56:03,036 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "058c0dbe-7061-419a-804d-605a1c1b6341" 2024-11-22T18:56:03,037 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f7bc509, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:56:03,037 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d79ba0c344fb,44109,-1] 2024-11-22T18:56:03,037 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T18:56:03,037 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:56:03,038 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51226, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T18:56:03,038 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@233af0ee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:56:03,039 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b465b92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:56:03,039 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d79ba0c344fb,44109,-1 for getting cluster id 2024-11-22T18:56:03,039 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T18:56:03,040 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '058c0dbe-7061-419a-804d-605a1c1b6341' 2024-11-22T18:56:03,040 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T18:56:03,040 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "058c0dbe-7061-419a-804d-605a1c1b6341" 2024-11-22T18:56:03,040 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ba83d75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:56:03,040 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d79ba0c344fb,44109,-1] 2024-11-22T18:56:03,040 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T18:56:03,041 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:56:03,041 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51244, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T18:56:03,042 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e2769e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:56:03,042 DEBUG [Thread-623 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T18:56:03,042 DEBUG [Thread-616 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T18:56:03,043 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T18:56:03,043 DEBUG [Thread-625 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T18:56:03,043 DEBUG [Thread-629 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T18:56:03,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d79ba0c344fb,43593,1732301724173, seqNum=-1] 2024-11-22T18:56:03,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d79ba0c344fb,43593,1732301724173, seqNum=-1] 2024-11-22T18:56:03,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-22T18:56:03,044 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d79ba0c344fb,43593,1732301724173, seqNum=-1] 2024-11-22T18:56:03,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T18:56:03,044 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T18:56:03,044 DEBUG [Thread-627 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T18:56:03,045 DEBUG [Thread-614 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T18:56:03,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T18:56:03,045 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T18:56:03,046 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43426, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T18:56:03,046 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43430, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T18:56:03,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d79ba0c344fb,43593,1732301724173, seqNum=-1] 2024-11-22T18:56:03,046 DEBUG [Thread-620 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T18:56:03,046 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d79ba0c344fb,43593,1732301724173, seqNum=-1] 2024-11-22T18:56:03,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T18:56:03,047 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T18:56:03,047 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d79ba0c344fb,43593,1732301724173, seqNum=-1] 2024-11-22T18:56:03,047 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T18:56:03,047 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T18:56:03,047 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T18:56:03,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-22T18:56:03,048 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43452, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T18:56:03,048 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43440, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T18:56:03,048 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43464, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T18:56:03,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d79ba0c344fb,43593,1732301724173, seqNum=-1] 2024-11-22T18:56:03,049 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43470, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T18:56:03,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T18:56:03,050 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43478, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T18:56:03,051 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5] 2024-11-22T18:56:03,052 DEBUG [Thread-612 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T18:56:03,052 DEBUG [Thread-618 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T18:56:03,054 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5] 2024-11-22T18:56:03,054 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5] 2024-11-22T18:56:03,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d79ba0c344fb,43593,1732301724173, seqNum=-1] 2024-11-22T18:56:03,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T18:56:03,055 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5] 2024-11-22T18:56:03,055 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d79ba0c344fb,43593,1732301724173, seqNum=-1] 2024-11-22T18:56:03,055 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T18:56:03,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5] 2024-11-22T18:56:03,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5] 2024-11-22T18:56:03,057 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5] 2024-11-22T18:56:03,057 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43494, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T18:56:03,058 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43508, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T18:56:03,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5] 2024-11-22T18:56:03,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5] 2024-11-22T18:56:03,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:03,077 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T18:56:03,077 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e2eea2cf5b56baf2f173c11708a11448 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-22T18:56:03,118 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112276b35f65a2b046e2aa2792dad32c5c50_e2eea2cf5b56baf2f173c11708a11448 is 50, key is test_row_0/A:col10/1732301763071/Put/seqid=0 2024-11-22T18:56:03,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741945_1121 (size=17034) 2024-11-22T18:56:03,136 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:56:03,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:03,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43478 deadline: 1732301823136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:03,143 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112276b35f65a2b046e2aa2792dad32c5c50_e2eea2cf5b56baf2f173c11708a11448 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112276b35f65a2b046e2aa2792dad32c5c50_e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:03,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:03,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:03,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:03,145 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/403ce19885a84436976f47c5f56abe64, store: [table=TestAcidGuarantees family=A region=e2eea2cf5b56baf2f173c11708a11448] 2024-11-22T18:56:03,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:03,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43470 deadline: 1732301823139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:03,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:03,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:03,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:03,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43494 deadline: 1732301823139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:03,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:03,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:03,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:03,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:03,154 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/403ce19885a84436976f47c5f56abe64 is 175, key is test_row_0/A:col10/1732301763071/Put/seqid=0 2024-11-22T18:56:03,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-22T18:56:03,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:03,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43426 deadline: 1732301823143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:03,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:03,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43508 deadline: 1732301823144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:03,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:03,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:03,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:03,166 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:03,166 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:03,166 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:03,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741946_1122 (size=48139) 2024-11-22T18:56:03,173 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=17, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/403ce19885a84436976f47c5f56abe64 2024-11-22T18:56:03,204 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-22T18:56:03,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:03,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. as already flushing 2024-11-22T18:56:03,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:03,204 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:03,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:03,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:03,208 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/B/138af5913f8847bf9addf3a422eab171 is 50, key is test_row_0/B:col10/1732301763071/Put/seqid=0 2024-11-22T18:56:03,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741947_1123 (size=12001) 2024-11-22T18:56:03,224 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/B/138af5913f8847bf9addf3a422eab171 2024-11-22T18:56:03,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:03,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43478 deadline: 1732301823246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:03,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:03,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:03,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:03,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:03,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43470 deadline: 1732301823256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:03,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:03,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43494 deadline: 1732301823256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:03,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:03,261 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:03,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:03,261 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:03,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:03,261 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:03,263 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/C/41d539f400c44763956b04a79a5acd13 is 50, key is test_row_0/C:col10/1732301763071/Put/seqid=0 2024-11-22T18:56:03,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:03,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43508 deadline: 1732301823265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:03,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:03,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:03,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:03,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:03,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43426 deadline: 1732301823276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:03,280 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:03,280 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:03,280 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:03,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741948_1124 (size=12001) 2024-11-22T18:56:03,285 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/C/41d539f400c44763956b04a79a5acd13 2024-11-22T18:56:03,292 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/403ce19885a84436976f47c5f56abe64 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/403ce19885a84436976f47c5f56abe64 2024-11-22T18:56:03,299 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/403ce19885a84436976f47c5f56abe64, entries=250, sequenceid=17, filesize=47.0 K 2024-11-22T18:56:03,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/B/138af5913f8847bf9addf3a422eab171 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/138af5913f8847bf9addf3a422eab171 2024-11-22T18:56:03,308 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/138af5913f8847bf9addf3a422eab171, entries=150, sequenceid=17, filesize=11.7 K 2024-11-22T18:56:03,309 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/C/41d539f400c44763956b04a79a5acd13 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/41d539f400c44763956b04a79a5acd13 2024-11-22T18:56:03,316 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/41d539f400c44763956b04a79a5acd13, entries=150, sequenceid=17, filesize=11.7 K 2024-11-22T18:56:03,317 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for e2eea2cf5b56baf2f173c11708a11448 in 240ms, sequenceid=17, compaction requested=false 2024-11-22T18:56:03,318 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-22T18:56:03,318 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e2eea2cf5b56baf2f173c11708a11448: 2024-11-22T18:56:03,359 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-22T18:56:03,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:03,359 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing e2eea2cf5b56baf2f173c11708a11448 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-22T18:56:03,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-22T18:56:03,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112208e568be8bee427cbdceb1f8e6287346_e2eea2cf5b56baf2f173c11708a11448 is 50, key is test_row_0/A:col10/1732301763134/Put/seqid=0 2024-11-22T18:56:03,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741949_1125 (size=12154) 2024-11-22T18:56:03,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:56:03,392 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112208e568be8bee427cbdceb1f8e6287346_e2eea2cf5b56baf2f173c11708a11448 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112208e568be8bee427cbdceb1f8e6287346_e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:03,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/28d67d7ebd594de5bb52acf3ee65ba3d, store: [table=TestAcidGuarantees family=A region=e2eea2cf5b56baf2f173c11708a11448] 2024-11-22T18:56:03,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/28d67d7ebd594de5bb52acf3ee65ba3d is 175, key is test_row_0/A:col10/1732301763134/Put/seqid=0 2024-11-22T18:56:03,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741950_1126 (size=30955) 2024-11-22T18:56:03,410 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/28d67d7ebd594de5bb52acf3ee65ba3d 2024-11-22T18:56:03,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/B/4e80a32fabf045839b400a806e9d736c is 50, key is test_row_0/B:col10/1732301763134/Put/seqid=0 2024-11-22T18:56:03,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741951_1127 (size=12001) 2024-11-22T18:56:03,435 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/B/4e80a32fabf045839b400a806e9d736c 2024-11-22T18:56:03,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/C/8918033e41ea4dab845f8675e3613969 is 50, key is test_row_0/C:col10/1732301763134/Put/seqid=0 2024-11-22T18:56:03,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:03,462 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. as already flushing 2024-11-22T18:56:03,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741952_1128 (size=12001) 2024-11-22T18:56:03,481 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/C/8918033e41ea4dab845f8675e3613969 2024-11-22T18:56:03,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/28d67d7ebd594de5bb52acf3ee65ba3d as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/28d67d7ebd594de5bb52acf3ee65ba3d 2024-11-22T18:56:03,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:03,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43470 deadline: 1732301823486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:03,493 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:03,493 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:03,493 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:03,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:03,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43426 deadline: 1732301823486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:03,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:03,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43508 deadline: 1732301823487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:03,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:03,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43478 deadline: 1732301823488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:03,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:03,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:03,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:03,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:03,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43494 deadline: 1732301823489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:03,494 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:03,494 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:03,494 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:03,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:03,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:03,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:03,495 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:03,495 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:03,495 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:03,501 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/28d67d7ebd594de5bb52acf3ee65ba3d, entries=150, sequenceid=40, filesize=30.2 K 2024-11-22T18:56:03,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/B/4e80a32fabf045839b400a806e9d736c as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/4e80a32fabf045839b400a806e9d736c 2024-11-22T18:56:03,508 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/4e80a32fabf045839b400a806e9d736c, entries=150, sequenceid=40, filesize=11.7 K 2024-11-22T18:56:03,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/C/8918033e41ea4dab845f8675e3613969 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/8918033e41ea4dab845f8675e3613969 2024-11-22T18:56:03,514 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/8918033e41ea4dab845f8675e3613969, entries=150, sequenceid=40, filesize=11.7 K 2024-11-22T18:56:03,516 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for e2eea2cf5b56baf2f173c11708a11448 in 157ms, sequenceid=40, compaction requested=false 2024-11-22T18:56:03,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for e2eea2cf5b56baf2f173c11708a11448: 2024-11-22T18:56:03,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:03,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-22T18:56:03,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-22T18:56:03,521 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-22T18:56:03,521 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 471 msec 2024-11-22T18:56:03,523 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 479 msec 2024-11-22T18:56:03,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:03,598 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T18:56:03,599 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e2eea2cf5b56baf2f173c11708a11448 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-22T18:56:03,614 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122ea09690e921d46838a5c31cefad8bd6f_e2eea2cf5b56baf2f173c11708a11448 is 50, key is test_row_0/A:col10/1732301763596/Put/seqid=0 2024-11-22T18:56:03,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741953_1129 (size=14594) 2024-11-22T18:56:03,628 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:56:03,635 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122ea09690e921d46838a5c31cefad8bd6f_e2eea2cf5b56baf2f173c11708a11448 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122ea09690e921d46838a5c31cefad8bd6f_e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:03,637 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/1536a8e4f6244044a3a8775d1433cd72, store: [table=TestAcidGuarantees family=A region=e2eea2cf5b56baf2f173c11708a11448] 2024-11-22T18:56:03,637 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/1536a8e4f6244044a3a8775d1433cd72 is 175, key is test_row_0/A:col10/1732301763596/Put/seqid=0 2024-11-22T18:56:03,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741954_1130 (size=39549) 2024-11-22T18:56:03,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-22T18:56:03,676 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T18:56:03,677 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T18:56:03,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-22T18:56:03,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-22T18:56:03,679 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T18:56:03,680 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T18:56:03,680 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T18:56:03,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:03,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43470 deadline: 1732301823715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:03,725 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:03,725 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:03,725 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:03,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:03,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43508 deadline: 1732301823717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:03,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:03,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:03,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:03,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:03,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43478 deadline: 1732301823723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:03,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:03,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:03,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43494 deadline: 1732301823725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:03,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:03,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:03,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:03,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:03,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:03,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-22T18:56:03,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:03,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43426 deadline: 1732301823805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:03,811 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:03,811 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:03,811 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:03,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:03,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43478 deadline: 1732301823836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:03,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:03,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:03,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43494 deadline: 1732301823836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:03,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:03,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:03,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:03,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:03,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:03,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:03,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43470 deadline: 1732301823836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:03,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:03,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:03,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:03,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43508 deadline: 1732301823836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:03,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:03,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:03,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:03,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:03,854 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-22T18:56:03,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:03,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. as already flushing 2024-11-22T18:56:03,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:03,854 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:03,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:03,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:03,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-22T18:56:04,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-22T18:56:04,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:04,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. as already flushing 2024-11-22T18:56:04,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:04,009 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:04,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:04,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:04,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:04,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43508 deadline: 1732301824046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:04,055 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=54, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/1536a8e4f6244044a3a8775d1433cd72 2024-11-22T18:56:04,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:04,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43494 deadline: 1732301824047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:04,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:04,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43470 deadline: 1732301824047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:04,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:04,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43478 deadline: 1732301824047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:04,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:04,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:04,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:04,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:04,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:04,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:04,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:04,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:04,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:04,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:04,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:04,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:04,068 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/B/ec421512489d41989ba0939732a9b363 is 50, key is test_row_0/B:col10/1732301763596/Put/seqid=0 2024-11-22T18:56:04,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741955_1131 (size=12001) 2024-11-22T18:56:04,084 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/B/ec421512489d41989ba0939732a9b363 2024-11-22T18:56:04,094 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/C/846a945de0814246b5321b24117dd757 is 50, key is test_row_0/C:col10/1732301763596/Put/seqid=0 2024-11-22T18:56:04,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741956_1132 (size=12001) 2024-11-22T18:56:04,132 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/C/846a945de0814246b5321b24117dd757 2024-11-22T18:56:04,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/1536a8e4f6244044a3a8775d1433cd72 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/1536a8e4f6244044a3a8775d1433cd72 2024-11-22T18:56:04,149 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/1536a8e4f6244044a3a8775d1433cd72, entries=200, sequenceid=54, filesize=38.6 K 2024-11-22T18:56:04,150 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/B/ec421512489d41989ba0939732a9b363 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/ec421512489d41989ba0939732a9b363 2024-11-22T18:56:04,158 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/ec421512489d41989ba0939732a9b363, entries=150, sequenceid=54, filesize=11.7 K 2024-11-22T18:56:04,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/C/846a945de0814246b5321b24117dd757 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/846a945de0814246b5321b24117dd757 2024-11-22T18:56:04,167 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/846a945de0814246b5321b24117dd757, entries=150, sequenceid=54, filesize=11.7 K 2024-11-22T18:56:04,168 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-22T18:56:04,168 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for e2eea2cf5b56baf2f173c11708a11448 in 570ms, sequenceid=54, compaction requested=true 2024-11-22T18:56:04,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e2eea2cf5b56baf2f173c11708a11448: 2024-11-22T18:56:04,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:04,168 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T18:56:04,169 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing e2eea2cf5b56baf2f173c11708a11448 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-22T18:56:04,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e2eea2cf5b56baf2f173c11708a11448:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T18:56:04,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:56:04,169 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T18:56:04,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e2eea2cf5b56baf2f173c11708a11448:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T18:56:04,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:56:04,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e2eea2cf5b56baf2f173c11708a11448:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T18:56:04,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:56:04,170 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T18:56:04,171 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1541): e2eea2cf5b56baf2f173c11708a11448/A is initiating minor compaction (all files) 2024-11-22T18:56:04,171 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e2eea2cf5b56baf2f173c11708a11448/A in TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:04,171 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/403ce19885a84436976f47c5f56abe64, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/28d67d7ebd594de5bb52acf3ee65ba3d, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/1536a8e4f6244044a3a8775d1433cd72] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp, totalSize=115.9 K 2024-11-22T18:56:04,171 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:04,171 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. files: [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/403ce19885a84436976f47c5f56abe64, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/28d67d7ebd594de5bb52acf3ee65ba3d, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/1536a8e4f6244044a3a8775d1433cd72] 2024-11-22T18:56:04,171 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T18:56:04,172 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1541): e2eea2cf5b56baf2f173c11708a11448/B is initiating minor compaction (all files) 2024-11-22T18:56:04,172 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e2eea2cf5b56baf2f173c11708a11448/B in TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:04,172 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/138af5913f8847bf9addf3a422eab171, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/4e80a32fabf045839b400a806e9d736c, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/ec421512489d41989ba0939732a9b363] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp, totalSize=35.2 K 2024-11-22T18:56:04,174 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 138af5913f8847bf9addf3a422eab171, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1732301763070 2024-11-22T18:56:04,174 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 403ce19885a84436976f47c5f56abe64, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1732301763062 2024-11-22T18:56:04,175 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e80a32fabf045839b400a806e9d736c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732301763104 2024-11-22T18:56:04,175 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28d67d7ebd594de5bb52acf3ee65ba3d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732301763104 2024-11-22T18:56:04,175 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting ec421512489d41989ba0939732a9b363, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732301763484 2024-11-22T18:56:04,176 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1536a8e4f6244044a3a8775d1433cd72, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732301763476 2024-11-22T18:56:04,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122ba28e75f72784bbf8e45f6dc21980c64_e2eea2cf5b56baf2f173c11708a11448 is 50, key is test_row_0/A:col10/1732301763640/Put/seqid=0 2024-11-22T18:56:04,202 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e2eea2cf5b56baf2f173c11708a11448#B#compaction#115 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:56:04,202 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/B/c2d87713cc8a4848b9f9aa047d033938 is 50, key is test_row_0/B:col10/1732301763596/Put/seqid=0 2024-11-22T18:56:04,204 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e2eea2cf5b56baf2f173c11708a11448] 2024-11-22T18:56:04,229 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411221eed11d4b6a5449f92b21bb5f28fe2f9_e2eea2cf5b56baf2f173c11708a11448 store=[table=TestAcidGuarantees family=A region=e2eea2cf5b56baf2f173c11708a11448] 2024-11-22T18:56:04,236 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411221eed11d4b6a5449f92b21bb5f28fe2f9_e2eea2cf5b56baf2f173c11708a11448, store=[table=TestAcidGuarantees family=A region=e2eea2cf5b56baf2f173c11708a11448] 2024-11-22T18:56:04,236 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411221eed11d4b6a5449f92b21bb5f28fe2f9_e2eea2cf5b56baf2f173c11708a11448 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e2eea2cf5b56baf2f173c11708a11448] 2024-11-22T18:56:04,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741957_1133 (size=12154) 2024-11-22T18:56:04,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741958_1134 (size=12104) 2024-11-22T18:56:04,258 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/B/c2d87713cc8a4848b9f9aa047d033938 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/c2d87713cc8a4848b9f9aa047d033938 2024-11-22T18:56:04,265 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e2eea2cf5b56baf2f173c11708a11448/B of e2eea2cf5b56baf2f173c11708a11448 into c2d87713cc8a4848b9f9aa047d033938(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:56:04,265 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e2eea2cf5b56baf2f173c11708a11448: 2024-11-22T18:56:04,265 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., storeName=e2eea2cf5b56baf2f173c11708a11448/B, priority=13, startTime=1732301764169; duration=0sec 2024-11-22T18:56:04,266 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:56:04,266 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e2eea2cf5b56baf2f173c11708a11448:B 2024-11-22T18:56:04,266 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T18:56:04,267 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T18:56:04,267 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1541): e2eea2cf5b56baf2f173c11708a11448/C is initiating minor compaction (all files) 2024-11-22T18:56:04,267 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e2eea2cf5b56baf2f173c11708a11448/C in TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:04,267 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/41d539f400c44763956b04a79a5acd13, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/8918033e41ea4dab845f8675e3613969, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/846a945de0814246b5321b24117dd757] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp, totalSize=35.2 K 2024-11-22T18:56:04,268 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 41d539f400c44763956b04a79a5acd13, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1732301763070 2024-11-22T18:56:04,268 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 8918033e41ea4dab845f8675e3613969, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732301763104 2024-11-22T18:56:04,269 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 846a945de0814246b5321b24117dd757, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732301763484 2024-11-22T18:56:04,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741959_1135 (size=4469) 2024-11-22T18:56:04,279 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e2eea2cf5b56baf2f173c11708a11448#A#compaction#116 average throughput is 0.33 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:56:04,280 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/940fa24840b040d594a798075f846816 is 175, key is test_row_0/A:col10/1732301763596/Put/seqid=0 2024-11-22T18:56:04,295 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e2eea2cf5b56baf2f173c11708a11448#C#compaction#117 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:56:04,295 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/C/06d7d604bcbf4e11ab8dfe11bd7816bf is 50, key is test_row_0/C:col10/1732301763596/Put/seqid=0 2024-11-22T18:56:04,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-22T18:56:04,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741960_1136 (size=31058) 2024-11-22T18:56:04,316 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/940fa24840b040d594a798075f846816 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/940fa24840b040d594a798075f846816 2024-11-22T18:56:04,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741961_1137 (size=12104) 2024-11-22T18:56:04,325 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e2eea2cf5b56baf2f173c11708a11448/A of e2eea2cf5b56baf2f173c11708a11448 into 940fa24840b040d594a798075f846816(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:56:04,325 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e2eea2cf5b56baf2f173c11708a11448: 2024-11-22T18:56:04,325 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., storeName=e2eea2cf5b56baf2f173c11708a11448/A, priority=13, startTime=1732301764168; duration=0sec 2024-11-22T18:56:04,325 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:56:04,325 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e2eea2cf5b56baf2f173c11708a11448:A 2024-11-22T18:56:04,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:04,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. as already flushing 2024-11-22T18:56:04,335 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/C/06d7d604bcbf4e11ab8dfe11bd7816bf as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/06d7d604bcbf4e11ab8dfe11bd7816bf 2024-11-22T18:56:04,345 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e2eea2cf5b56baf2f173c11708a11448/C of e2eea2cf5b56baf2f173c11708a11448 into 06d7d604bcbf4e11ab8dfe11bd7816bf(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:56:04,346 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e2eea2cf5b56baf2f173c11708a11448: 2024-11-22T18:56:04,346 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., storeName=e2eea2cf5b56baf2f173c11708a11448/C, priority=13, startTime=1732301764170; duration=0sec 2024-11-22T18:56:04,346 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:56:04,346 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e2eea2cf5b56baf2f173c11708a11448:C 2024-11-22T18:56:04,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:04,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43426 deadline: 1732301824368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:04,373 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:04,373 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:04,373 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:04,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:04,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43508 deadline: 1732301824370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:04,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:04,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43494 deadline: 1732301824371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:04,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:04,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:04,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:04,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:04,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:04,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:04,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:04,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43470 deadline: 1732301824371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:04,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:04,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43478 deadline: 1732301824372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:04,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:04,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:04,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:04,379 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:04,379 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:04,379 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:04,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:04,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43426 deadline: 1732301824475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:04,478 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:04,478 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:04,478 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:04,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:04,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43470 deadline: 1732301824486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:04,488 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:04,488 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:04,488 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:04,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:04,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43508 deadline: 1732301824486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:04,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:04,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43478 deadline: 1732301824486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:04,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:04,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:04,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:04,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:04,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:04,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:04,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:04,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43494 deadline: 1732301824486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:04,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:04,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:04,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:04,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:56:04,650 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122ba28e75f72784bbf8e45f6dc21980c64_e2eea2cf5b56baf2f173c11708a11448 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122ba28e75f72784bbf8e45f6dc21980c64_e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:04,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/3debbb0a5d654b379d049f0a4e66664b, store: [table=TestAcidGuarantees family=A region=e2eea2cf5b56baf2f173c11708a11448] 2024-11-22T18:56:04,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/3debbb0a5d654b379d049f0a4e66664b is 175, key is test_row_0/A:col10/1732301763640/Put/seqid=0 2024-11-22T18:56:04,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741962_1138 (size=30955) 2024-11-22T18:56:04,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:04,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43426 deadline: 1732301824685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:04,688 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:04,688 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:04,688 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:04,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:04,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43470 deadline: 1732301824696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:04,697 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:04,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:04,697 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:04,697 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:04,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43508 deadline: 1732301824696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:04,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:04,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43478 deadline: 1732301824696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:04,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:04,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:04,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:04,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:04,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:04,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:04,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43494 deadline: 1732301824696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:04,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:04,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:04,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:04,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:04,727 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-22T18:56:04,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-22T18:56:04,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:04,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43426 deadline: 1732301824996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:04,998 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:04,998 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:04,998 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:05,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:05,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43470 deadline: 1732301825006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:05,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:05,011 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:05,011 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:05,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43508 deadline: 1732301825006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:05,011 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:05,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:05,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:05,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:05,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:05,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43478 deadline: 1732301825006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:05,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:05,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43494 deadline: 1732301825006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:05,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:05,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:05,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:05,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:05,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:05,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:05,058 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=76, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/3debbb0a5d654b379d049f0a4e66664b 2024-11-22T18:56:05,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/B/e621a93964d14ed5a6603552411c70a3 is 50, key is test_row_0/B:col10/1732301763640/Put/seqid=0 2024-11-22T18:56:05,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741963_1139 (size=12001) 2024-11-22T18:56:05,096 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/B/e621a93964d14ed5a6603552411c70a3 2024-11-22T18:56:05,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/C/209b6ce9d2434e268abbcebe479b388b is 50, key is test_row_0/C:col10/1732301763640/Put/seqid=0 2024-11-22T18:56:05,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741964_1140 (size=12001) 2024-11-22T18:56:05,116 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/C/209b6ce9d2434e268abbcebe479b388b 2024-11-22T18:56:05,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/3debbb0a5d654b379d049f0a4e66664b as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/3debbb0a5d654b379d049f0a4e66664b 2024-11-22T18:56:05,130 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/3debbb0a5d654b379d049f0a4e66664b, entries=150, sequenceid=76, filesize=30.2 K 2024-11-22T18:56:05,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/B/e621a93964d14ed5a6603552411c70a3 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/e621a93964d14ed5a6603552411c70a3 2024-11-22T18:56:05,137 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/e621a93964d14ed5a6603552411c70a3, entries=150, sequenceid=76, filesize=11.7 K 2024-11-22T18:56:05,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/C/209b6ce9d2434e268abbcebe479b388b as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/209b6ce9d2434e268abbcebe479b388b 2024-11-22T18:56:05,146 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/209b6ce9d2434e268abbcebe479b388b, entries=150, sequenceid=76, filesize=11.7 K 2024-11-22T18:56:05,148 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for e2eea2cf5b56baf2f173c11708a11448 in 979ms, sequenceid=76, compaction requested=false 2024-11-22T18:56:05,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for e2eea2cf5b56baf2f173c11708a11448: 2024-11-22T18:56:05,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:05,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-22T18:56:05,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-22T18:56:05,153 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-22T18:56:05,153 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4700 sec 2024-11-22T18:56:05,157 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.4780 sec 2024-11-22T18:56:05,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:05,508 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T18:56:05,508 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e2eea2cf5b56baf2f173c11708a11448 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-22T18:56:05,519 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122a03f438e4640474695b5f017be3728b6_e2eea2cf5b56baf2f173c11708a11448 is 50, key is test_row_0/A:col10/1732301765506/Put/seqid=0 2024-11-22T18:56:05,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741965_1141 (size=14594) 2024-11-22T18:56:05,535 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:56:05,544 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122a03f438e4640474695b5f017be3728b6_e2eea2cf5b56baf2f173c11708a11448 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122a03f438e4640474695b5f017be3728b6_e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:05,546 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/fdffb78f50654d9d906b180742e0d74c, store: [table=TestAcidGuarantees family=A region=e2eea2cf5b56baf2f173c11708a11448] 2024-11-22T18:56:05,546 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/fdffb78f50654d9d906b180742e0d74c is 175, key is test_row_0/A:col10/1732301765506/Put/seqid=0 2024-11-22T18:56:05,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:05,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43470 deadline: 1732301825545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:05,555 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:05,555 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:05,555 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:05,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741966_1142 (size=39549) 2024-11-22T18:56:05,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:05,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43508 deadline: 1732301825551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:05,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:05,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:05,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:05,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:05,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43426 deadline: 1732301825553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:05,558 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=95, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/fdffb78f50654d9d906b180742e0d74c 2024-11-22T18:56:05,558 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:05,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:05,558 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:05,559 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:05,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43494 deadline: 1732301825553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:05,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:05,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:05,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43478 deadline: 1732301825555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:05,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:05,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:05,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:05,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:05,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:05,570 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/B/4bf4f02a9d39482fbdae70c98d19ad8f is 50, key is test_row_0/B:col10/1732301765506/Put/seqid=0 2024-11-22T18:56:05,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741967_1143 (size=12001) 2024-11-22T18:56:05,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:05,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43470 deadline: 1732301825666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:05,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:05,669 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:05,669 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:05,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43508 deadline: 1732301825666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:05,669 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:05,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:05,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:05,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43426 deadline: 1732301825666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:05,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:05,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:05,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:05,670 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:05,670 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:05,670 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:05,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43494 deadline: 1732301825666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:05,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:05,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43478 deadline: 1732301825666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:05,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:05,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:05,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:05,670 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:05,670 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:05,670 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:05,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-22T18:56:05,826 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T18:56:05,826 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T18:56:05,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-22T18:56:05,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-22T18:56:05,828 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T18:56:05,830 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T18:56:05,830 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T18:56:05,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:05,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43470 deadline: 1732301825875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:05,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:05,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43508 deadline: 1732301825875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:05,877 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:05,877 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:05,877 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:05,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:05,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:05,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:05,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43426 deadline: 1732301825875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:05,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:05,878 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:05,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:05,878 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:05,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43494 deadline: 1732301825875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:05,878 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:05,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:05,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43478 deadline: 1732301825876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:05,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:05,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:05,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:05,879 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:05,879 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:05,879 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:05,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-22T18:56:05,983 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-22T18:56:05,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:05,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. as already flushing 2024-11-22T18:56:05,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:05,983 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:05,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:05,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:05,988 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/B/4bf4f02a9d39482fbdae70c98d19ad8f 2024-11-22T18:56:05,997 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/C/3362a1202bfa4edaae5daf7378c8ce98 is 50, key is test_row_0/C:col10/1732301765506/Put/seqid=0 2024-11-22T18:56:06,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741968_1144 (size=12001) 2024-11-22T18:56:06,136 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-22T18:56:06,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:06,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. as already flushing 2024-11-22T18:56:06,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:06,137 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:06,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:06,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:06,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-22T18:56:06,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:06,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43470 deadline: 1732301826185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:06,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:06,187 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:06,187 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:06,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43508 deadline: 1732301826185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:06,187 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:06,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:06,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:06,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43426 deadline: 1732301826185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:06,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:06,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:06,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:06,188 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:06,188 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:06,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43494 deadline: 1732301826185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:06,188 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:06,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:06,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:06,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:06,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43478 deadline: 1732301826185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:06,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:06,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:06,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:06,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:06,290 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-22T18:56:06,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:06,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. as already flushing 2024-11-22T18:56:06,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:06,290 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:06,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:06,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:06,401 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/C/3362a1202bfa4edaae5daf7378c8ce98 2024-11-22T18:56:06,407 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/fdffb78f50654d9d906b180742e0d74c as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/fdffb78f50654d9d906b180742e0d74c 2024-11-22T18:56:06,413 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/fdffb78f50654d9d906b180742e0d74c, entries=200, sequenceid=95, filesize=38.6 K 2024-11-22T18:56:06,414 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/B/4bf4f02a9d39482fbdae70c98d19ad8f as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/4bf4f02a9d39482fbdae70c98d19ad8f 2024-11-22T18:56:06,419 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/4bf4f02a9d39482fbdae70c98d19ad8f, entries=150, sequenceid=95, filesize=11.7 K 2024-11-22T18:56:06,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/C/3362a1202bfa4edaae5daf7378c8ce98 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/3362a1202bfa4edaae5daf7378c8ce98 2024-11-22T18:56:06,425 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/3362a1202bfa4edaae5daf7378c8ce98, entries=150, sequenceid=95, filesize=11.7 K 2024-11-22T18:56:06,427 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for e2eea2cf5b56baf2f173c11708a11448 in 919ms, sequenceid=95, compaction requested=true 2024-11-22T18:56:06,427 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e2eea2cf5b56baf2f173c11708a11448: 2024-11-22T18:56:06,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e2eea2cf5b56baf2f173c11708a11448:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T18:56:06,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:56:06,427 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T18:56:06,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e2eea2cf5b56baf2f173c11708a11448:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T18:56:06,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:56:06,427 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T18:56:06,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e2eea2cf5b56baf2f173c11708a11448:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T18:56:06,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:56:06,429 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T18:56:06,429 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1541): e2eea2cf5b56baf2f173c11708a11448/A is initiating minor compaction (all files) 2024-11-22T18:56:06,429 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e2eea2cf5b56baf2f173c11708a11448/A in TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:06,429 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/940fa24840b040d594a798075f846816, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/3debbb0a5d654b379d049f0a4e66664b, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/fdffb78f50654d9d906b180742e0d74c] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp, totalSize=99.2 K 2024-11-22T18:56:06,429 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:06,429 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. files: [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/940fa24840b040d594a798075f846816, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/3debbb0a5d654b379d049f0a4e66664b, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/fdffb78f50654d9d906b180742e0d74c] 2024-11-22T18:56:06,430 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T18:56:06,430 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1541): e2eea2cf5b56baf2f173c11708a11448/B is initiating minor compaction (all files) 2024-11-22T18:56:06,430 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e2eea2cf5b56baf2f173c11708a11448/B in TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:06,430 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/c2d87713cc8a4848b9f9aa047d033938, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/e621a93964d14ed5a6603552411c70a3, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/4bf4f02a9d39482fbdae70c98d19ad8f] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp, totalSize=35.3 K 2024-11-22T18:56:06,431 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 940fa24840b040d594a798075f846816, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732301763484 2024-11-22T18:56:06,431 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting c2d87713cc8a4848b9f9aa047d033938, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732301763484 2024-11-22T18:56:06,431 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3debbb0a5d654b379d049f0a4e66664b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732301763640 2024-11-22T18:56:06,431 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting e621a93964d14ed5a6603552411c70a3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732301763640 2024-11-22T18:56:06,432 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting fdffb78f50654d9d906b180742e0d74c, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732301764332 2024-11-22T18:56:06,432 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 4bf4f02a9d39482fbdae70c98d19ad8f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732301764332 2024-11-22T18:56:06,443 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-22T18:56:06,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:06,444 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing e2eea2cf5b56baf2f173c11708a11448 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-22T18:56:06,447 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e2eea2cf5b56baf2f173c11708a11448] 2024-11-22T18:56:06,450 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e2eea2cf5b56baf2f173c11708a11448#B#compaction#124 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:56:06,451 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/B/c7b303c54b8c44889eb06ec4cff6c487 is 50, key is test_row_0/B:col10/1732301765506/Put/seqid=0 2024-11-22T18:56:06,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-22T18:56:06,464 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411229f6281490ce2499cbccb48973ce64a2d_e2eea2cf5b56baf2f173c11708a11448 store=[table=TestAcidGuarantees family=A region=e2eea2cf5b56baf2f173c11708a11448] 2024-11-22T18:56:06,467 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411229f6281490ce2499cbccb48973ce64a2d_e2eea2cf5b56baf2f173c11708a11448, store=[table=TestAcidGuarantees family=A region=e2eea2cf5b56baf2f173c11708a11448] 2024-11-22T18:56:06,469 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411229f6281490ce2499cbccb48973ce64a2d_e2eea2cf5b56baf2f173c11708a11448 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e2eea2cf5b56baf2f173c11708a11448] 2024-11-22T18:56:06,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411228bd39e0c6416411e9e5d594f3624e836_e2eea2cf5b56baf2f173c11708a11448 is 50, key is test_row_0/A:col10/1732301765552/Put/seqid=0 2024-11-22T18:56:06,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741970_1146 (size=4469) 2024-11-22T18:56:06,478 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e2eea2cf5b56baf2f173c11708a11448#A#compaction#123 average throughput is 0.79 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:56:06,479 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/064a04273c704727917d3c19e1f6c55c is 175, key is test_row_0/A:col10/1732301765506/Put/seqid=0 2024-11-22T18:56:06,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741969_1145 (size=12207) 2024-11-22T18:56:06,494 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/B/c7b303c54b8c44889eb06ec4cff6c487 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/c7b303c54b8c44889eb06ec4cff6c487 2024-11-22T18:56:06,504 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e2eea2cf5b56baf2f173c11708a11448/B of e2eea2cf5b56baf2f173c11708a11448 into c7b303c54b8c44889eb06ec4cff6c487(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:56:06,504 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e2eea2cf5b56baf2f173c11708a11448: 2024-11-22T18:56:06,504 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., storeName=e2eea2cf5b56baf2f173c11708a11448/B, priority=13, startTime=1732301766427; duration=0sec 2024-11-22T18:56:06,505 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:56:06,505 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e2eea2cf5b56baf2f173c11708a11448:B 2024-11-22T18:56:06,505 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T18:56:06,507 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T18:56:06,507 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1541): e2eea2cf5b56baf2f173c11708a11448/C is initiating minor compaction (all files) 2024-11-22T18:56:06,507 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e2eea2cf5b56baf2f173c11708a11448/C in TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:06,507 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/06d7d604bcbf4e11ab8dfe11bd7816bf, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/209b6ce9d2434e268abbcebe479b388b, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/3362a1202bfa4edaae5daf7378c8ce98] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp, totalSize=35.3 K 2024-11-22T18:56:06,508 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 06d7d604bcbf4e11ab8dfe11bd7816bf, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732301763484 2024-11-22T18:56:06,508 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 209b6ce9d2434e268abbcebe479b388b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732301763640 2024-11-22T18:56:06,513 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 3362a1202bfa4edaae5daf7378c8ce98, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732301764332 2024-11-22T18:56:06,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741972_1148 (size=31161) 2024-11-22T18:56:06,542 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/064a04273c704727917d3c19e1f6c55c as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/064a04273c704727917d3c19e1f6c55c 2024-11-22T18:56:06,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741971_1147 (size=12154) 2024-11-22T18:56:06,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:56:06,550 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411228bd39e0c6416411e9e5d594f3624e836_e2eea2cf5b56baf2f173c11708a11448 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411228bd39e0c6416411e9e5d594f3624e836_e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:06,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/a67d1b35cae8491e864393b25dd6cf59, store: [table=TestAcidGuarantees family=A region=e2eea2cf5b56baf2f173c11708a11448] 2024-11-22T18:56:06,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/a67d1b35cae8491e864393b25dd6cf59 is 175, key is test_row_0/A:col10/1732301765552/Put/seqid=0 2024-11-22T18:56:06,552 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e2eea2cf5b56baf2f173c11708a11448/A of e2eea2cf5b56baf2f173c11708a11448 into 064a04273c704727917d3c19e1f6c55c(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:56:06,553 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e2eea2cf5b56baf2f173c11708a11448: 2024-11-22T18:56:06,553 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., storeName=e2eea2cf5b56baf2f173c11708a11448/A, priority=13, startTime=1732301766427; duration=0sec 2024-11-22T18:56:06,553 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:56:06,553 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e2eea2cf5b56baf2f173c11708a11448:A 2024-11-22T18:56:06,553 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e2eea2cf5b56baf2f173c11708a11448#C#compaction#126 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:56:06,554 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/C/16e4272972eb4ead8fda36217b16b356 is 50, key is test_row_0/C:col10/1732301765506/Put/seqid=0 2024-11-22T18:56:06,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741973_1149 (size=30955) 2024-11-22T18:56:06,577 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=115, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/a67d1b35cae8491e864393b25dd6cf59 2024-11-22T18:56:06,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741974_1150 (size=12207) 2024-11-22T18:56:06,593 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/C/16e4272972eb4ead8fda36217b16b356 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/16e4272972eb4ead8fda36217b16b356 2024-11-22T18:56:06,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/B/daba81a966e94ddab26f7e1ce57bfd38 is 50, key is test_row_0/B:col10/1732301765552/Put/seqid=0 2024-11-22T18:56:06,602 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e2eea2cf5b56baf2f173c11708a11448/C of e2eea2cf5b56baf2f173c11708a11448 into 16e4272972eb4ead8fda36217b16b356(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:56:06,602 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e2eea2cf5b56baf2f173c11708a11448: 2024-11-22T18:56:06,602 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., storeName=e2eea2cf5b56baf2f173c11708a11448/C, priority=13, startTime=1732301766427; duration=0sec 2024-11-22T18:56:06,603 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:56:06,603 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e2eea2cf5b56baf2f173c11708a11448:C 2024-11-22T18:56:06,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741975_1151 (size=12001) 2024-11-22T18:56:06,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:06,698 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. as already flushing 2024-11-22T18:56:06,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:06,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:06,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43470 deadline: 1732301826714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:06,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43494 deadline: 1732301826713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:06,718 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:06,718 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:06,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:06,718 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:06,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43478 deadline: 1732301826714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:06,719 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:06,719 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:06,719 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:06,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:06,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43426 deadline: 1732301826718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:06,719 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:06,719 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:06,720 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:06,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:06,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43508 deadline: 1732301826720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:06,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:06,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:06,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:06,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:06,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:06,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:06,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:06,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43470 deadline: 1732301826826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:06,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:06,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:06,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:06,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:06,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43478 deadline: 1732301826826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:06,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:06,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:06,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43426 deadline: 1732301826826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:06,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:06,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:06,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:06,829 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:06,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43508 deadline: 1732301826826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:06,829 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:06,829 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:06,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:06,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:06,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:06,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:06,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43494 deadline: 1732301826826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:06,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:06,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:06,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:06,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-22T18:56:07,006 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/B/daba81a966e94ddab26f7e1ce57bfd38 2024-11-22T18:56:07,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/C/a6d069f36f9641eabee962b77fa8ad4b is 50, key is test_row_0/C:col10/1732301765552/Put/seqid=0 2024-11-22T18:56:07,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741976_1152 (size=12001) 2024-11-22T18:56:07,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:07,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43470 deadline: 1732301827036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:07,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:07,038 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:07,038 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:07,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43478 deadline: 1732301827036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:07,038 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:07,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:07,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:07,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:07,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43426 deadline: 1732301827036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:07,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:07,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:07,039 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:07,039 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:07,039 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:07,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43508 deadline: 1732301827036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:07,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:07,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:07,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:07,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:07,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43494 deadline: 1732301827036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:07,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:07,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:07,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:07,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:07,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43470 deadline: 1732301827346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:07,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:07,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:07,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43478 deadline: 1732301827346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:07,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:07,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:07,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:07,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:07,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:07,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43426 deadline: 1732301827346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:07,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:07,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43508 deadline: 1732301827346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:07,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:07,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:07,348 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:07,348 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:07,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:07,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43494 deadline: 1732301827346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:07,348 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:07,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:07,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:07,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:07,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:07,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:07,421 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/C/a6d069f36f9641eabee962b77fa8ad4b 2024-11-22T18:56:07,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/a67d1b35cae8491e864393b25dd6cf59 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/a67d1b35cae8491e864393b25dd6cf59 2024-11-22T18:56:07,430 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/a67d1b35cae8491e864393b25dd6cf59, entries=150, sequenceid=115, filesize=30.2 K 2024-11-22T18:56:07,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/B/daba81a966e94ddab26f7e1ce57bfd38 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/daba81a966e94ddab26f7e1ce57bfd38 2024-11-22T18:56:07,437 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/daba81a966e94ddab26f7e1ce57bfd38, entries=150, sequenceid=115, filesize=11.7 K 2024-11-22T18:56:07,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/C/a6d069f36f9641eabee962b77fa8ad4b as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/a6d069f36f9641eabee962b77fa8ad4b 2024-11-22T18:56:07,443 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/a6d069f36f9641eabee962b77fa8ad4b, entries=150, sequenceid=115, filesize=11.7 K 2024-11-22T18:56:07,445 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for e2eea2cf5b56baf2f173c11708a11448 in 1002ms, sequenceid=115, compaction requested=false 2024-11-22T18:56:07,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for e2eea2cf5b56baf2f173c11708a11448: 2024-11-22T18:56:07,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:07,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-22T18:56:07,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-22T18:56:07,449 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-22T18:56:07,449 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6170 sec 2024-11-22T18:56:07,450 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.6220 sec 2024-11-22T18:56:07,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:07,858 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T18:56:07,858 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e2eea2cf5b56baf2f173c11708a11448 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-22T18:56:07,867 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112234f70a8bdae94c2da4859a135e5fa5e1_e2eea2cf5b56baf2f173c11708a11448 is 50, key is test_row_0/A:col10/1732301766715/Put/seqid=0 2024-11-22T18:56:07,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:07,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43508 deadline: 1732301827874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:07,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:07,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43494 deadline: 1732301827876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:07,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:07,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:07,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:07,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:07,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:07,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:07,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741977_1153 (size=12304) 2024-11-22T18:56:07,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:07,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43470 deadline: 1732301827877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:07,883 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:07,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:07,883 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:07,883 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:07,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43478 deadline: 1732301827878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:07,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:07,883 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:07,884 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:07,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43426 deadline: 1732301827878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:07,884 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:07,884 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:07,884 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:07,884 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:07,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-22T18:56:07,976 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T18:56:07,976 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T18:56:07,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-22T18:56:07,978 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T18:56:07,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-22T18:56:07,979 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T18:56:07,979 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T18:56:07,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:07,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43494 deadline: 1732301827986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:07,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:07,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:07,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:07,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43508 deadline: 1732301827986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:07,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:07,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:07,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:07,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:07,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:07,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43478 deadline: 1732301827986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:07,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:07,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:07,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:07,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:07,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43470 deadline: 1732301827986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:07,990 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:07,990 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:07,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:07,990 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:07,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43426 deadline: 1732301827986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:07,990 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:07,990 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:07,990 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:08,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-22T18:56:08,131 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-22T18:56:08,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:08,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. as already flushing 2024-11-22T18:56:08,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:08,132 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:08,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:08,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:08,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:08,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43494 deadline: 1732301828196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:08,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:08,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:08,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:08,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43508 deadline: 1732301828196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:08,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:08,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:08,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:08,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:08,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43478 deadline: 1732301828196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:08,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:08,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:08,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:08,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43470 deadline: 1732301828196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:08,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:08,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:08,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:08,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:08,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:08,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43426 deadline: 1732301828196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:08,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:08,202 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:08,202 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:08,202 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:08,284 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:56:08,285 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-22T18:56:08,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:08,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. as already flushing 2024-11-22T18:56:08,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:08,285 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:08,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:08,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:08,289 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112234f70a8bdae94c2da4859a135e5fa5e1_e2eea2cf5b56baf2f173c11708a11448 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112234f70a8bdae94c2da4859a135e5fa5e1_e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:08,290 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/cfe89cd0dd704203a578114fbd0b43f7, store: [table=TestAcidGuarantees family=A region=e2eea2cf5b56baf2f173c11708a11448] 2024-11-22T18:56:08,291 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/cfe89cd0dd704203a578114fbd0b43f7 is 175, key is test_row_0/A:col10/1732301766715/Put/seqid=0 2024-11-22T18:56:08,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-22T18:56:08,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741978_1154 (size=31105) 2024-11-22T18:56:08,339 INFO [master/d79ba0c344fb:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-22T18:56:08,339 INFO [master/d79ba0c344fb:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-22T18:56:08,439 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-22T18:56:08,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:08,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. as already flushing 2024-11-22T18:56:08,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:08,439 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:08,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:08,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:08,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:08,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43478 deadline: 1732301828506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:08,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:08,511 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:08,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43470 deadline: 1732301828506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:08,511 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:08,511 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:08,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:08,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:08,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43494 deadline: 1732301828506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:08,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:08,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:08,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:08,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:08,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:08,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43508 deadline: 1732301828507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:08,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:08,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:08,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:08,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:08,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:08,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43426 deadline: 1732301828507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:08,512 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:08,512 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:08,512 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:08,592 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-22T18:56:08,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:08,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. as already flushing 2024-11-22T18:56:08,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:08,593 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:08,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:08,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:08,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-22T18:56:08,697 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=137, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/cfe89cd0dd704203a578114fbd0b43f7 2024-11-22T18:56:08,705 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/B/b12db7b782a64d8b8e7897248cd59d0a is 50, key is test_row_0/B:col10/1732301766715/Put/seqid=0 2024-11-22T18:56:08,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741979_1155 (size=12151) 2024-11-22T18:56:08,710 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/B/b12db7b782a64d8b8e7897248cd59d0a 2024-11-22T18:56:08,717 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/C/2e4447bd01ba46b69667d63409b5848a is 50, key is test_row_0/C:col10/1732301766715/Put/seqid=0 2024-11-22T18:56:08,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741980_1156 (size=12151) 2024-11-22T18:56:08,722 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/C/2e4447bd01ba46b69667d63409b5848a 2024-11-22T18:56:08,727 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/cfe89cd0dd704203a578114fbd0b43f7 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/cfe89cd0dd704203a578114fbd0b43f7 2024-11-22T18:56:08,731 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/cfe89cd0dd704203a578114fbd0b43f7, entries=150, sequenceid=137, filesize=30.4 K 2024-11-22T18:56:08,732 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/B/b12db7b782a64d8b8e7897248cd59d0a as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/b12db7b782a64d8b8e7897248cd59d0a 2024-11-22T18:56:08,737 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/b12db7b782a64d8b8e7897248cd59d0a, entries=150, sequenceid=137, filesize=11.9 K 2024-11-22T18:56:08,738 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/C/2e4447bd01ba46b69667d63409b5848a as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/2e4447bd01ba46b69667d63409b5848a 2024-11-22T18:56:08,743 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/2e4447bd01ba46b69667d63409b5848a, entries=150, sequenceid=137, filesize=11.9 K 2024-11-22T18:56:08,744 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=100.63 KB/103050 for e2eea2cf5b56baf2f173c11708a11448 in 886ms, sequenceid=137, compaction requested=true 2024-11-22T18:56:08,744 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e2eea2cf5b56baf2f173c11708a11448: 2024-11-22T18:56:08,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e2eea2cf5b56baf2f173c11708a11448:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T18:56:08,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:56:08,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e2eea2cf5b56baf2f173c11708a11448:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T18:56:08,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:56:08,744 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T18:56:08,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e2eea2cf5b56baf2f173c11708a11448:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T18:56:08,744 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T18:56:08,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:56:08,745 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93221 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T18:56:08,745 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1541): e2eea2cf5b56baf2f173c11708a11448/A is initiating minor compaction (all files) 2024-11-22T18:56:08,746 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e2eea2cf5b56baf2f173c11708a11448/A in TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:08,746 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/064a04273c704727917d3c19e1f6c55c, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/a67d1b35cae8491e864393b25dd6cf59, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/cfe89cd0dd704203a578114fbd0b43f7] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp, totalSize=91.0 K 2024-11-22T18:56:08,746 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:08,746 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. files: [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/064a04273c704727917d3c19e1f6c55c, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/a67d1b35cae8491e864393b25dd6cf59, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/cfe89cd0dd704203a578114fbd0b43f7] 2024-11-22T18:56:08,746 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-22T18:56:08,746 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T18:56:08,746 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 064a04273c704727917d3c19e1f6c55c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732301764332 2024-11-22T18:56:08,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:08,746 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1541): e2eea2cf5b56baf2f173c11708a11448/B is initiating minor compaction (all files) 2024-11-22T18:56:08,747 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e2eea2cf5b56baf2f173c11708a11448/B in TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:08,747 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing e2eea2cf5b56baf2f173c11708a11448 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-22T18:56:08,747 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/c7b303c54b8c44889eb06ec4cff6c487, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/daba81a966e94ddab26f7e1ce57bfd38, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/b12db7b782a64d8b8e7897248cd59d0a] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp, totalSize=35.5 K 2024-11-22T18:56:08,747 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting a67d1b35cae8491e864393b25dd6cf59, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732301765531 2024-11-22T18:56:08,747 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting c7b303c54b8c44889eb06ec4cff6c487, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732301764332 2024-11-22T18:56:08,748 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting daba81a966e94ddab26f7e1ce57bfd38, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732301765531 2024-11-22T18:56:08,748 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting cfe89cd0dd704203a578114fbd0b43f7, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732301766715 2024-11-22T18:56:08,749 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting b12db7b782a64d8b8e7897248cd59d0a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732301766715 2024-11-22T18:56:08,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411225c9ba74b11e04114a769072ec0419bed_e2eea2cf5b56baf2f173c11708a11448 is 50, key is test_row_0/A:col10/1732301767876/Put/seqid=0 2024-11-22T18:56:08,765 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e2eea2cf5b56baf2f173c11708a11448] 2024-11-22T18:56:08,767 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e2eea2cf5b56baf2f173c11708a11448#B#compaction#133 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:56:08,767 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/B/008449021d554a3d8ec2c6790c8c2a05 is 50, key is test_row_0/B:col10/1732301766715/Put/seqid=0 2024-11-22T18:56:08,769 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411226846f18c205f435a88dd6f7d86cae936_e2eea2cf5b56baf2f173c11708a11448 store=[table=TestAcidGuarantees family=A region=e2eea2cf5b56baf2f173c11708a11448] 2024-11-22T18:56:08,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741981_1157 (size=12304) 2024-11-22T18:56:08,772 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411226846f18c205f435a88dd6f7d86cae936_e2eea2cf5b56baf2f173c11708a11448, store=[table=TestAcidGuarantees family=A region=e2eea2cf5b56baf2f173c11708a11448] 2024-11-22T18:56:08,772 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411226846f18c205f435a88dd6f7d86cae936_e2eea2cf5b56baf2f173c11708a11448 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e2eea2cf5b56baf2f173c11708a11448] 2024-11-22T18:56:08,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741982_1158 (size=12459) 2024-11-22T18:56:08,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741983_1159 (size=4469) 2024-11-22T18:56:09,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:09,018 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. as already flushing 2024-11-22T18:56:09,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:09,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43470 deadline: 1732301829062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:09,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:09,072 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:09,072 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:09,072 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:09,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43478 deadline: 1732301829063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:09,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:09,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43494 deadline: 1732301829063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:09,072 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:09,072 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:09,072 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:09,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:09,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:09,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:09,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:09,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43426 deadline: 1732301829070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:09,078 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:09,078 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:09,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:09,078 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:09,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43508 deadline: 1732301829071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:09,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:09,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:09,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:09,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-22T18:56:09,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:56:09,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:09,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43470 deadline: 1732301829176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:09,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:09,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:09,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43478 deadline: 1732301829176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:09,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:09,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:09,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:09,177 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:09,177 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:09,177 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:09,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43494 deadline: 1732301829176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:09,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:09,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:09,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:09,178 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411225c9ba74b11e04114a769072ec0419bed_e2eea2cf5b56baf2f173c11708a11448 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411225c9ba74b11e04114a769072ec0419bed_e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:09,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/2852d5900da449429f6be52be2dc9a29, store: [table=TestAcidGuarantees family=A region=e2eea2cf5b56baf2f173c11708a11448] 2024-11-22T18:56:09,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/2852d5900da449429f6be52be2dc9a29 is 175, key is test_row_0/A:col10/1732301767876/Put/seqid=0 2024-11-22T18:56:09,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741984_1160 (size=31105) 2024-11-22T18:56:09,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:09,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43426 deadline: 1732301829185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:09,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:09,188 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:09,188 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:09,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43508 deadline: 1732301829185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:09,188 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:09,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:09,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:09,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:09,191 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e2eea2cf5b56baf2f173c11708a11448#A#compaction#134 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:56:09,191 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/4a4b68be641a4316a96dad951bab0a1d is 175, key is test_row_0/A:col10/1732301766715/Put/seqid=0 2024-11-22T18:56:09,193 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/B/008449021d554a3d8ec2c6790c8c2a05 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/008449021d554a3d8ec2c6790c8c2a05 2024-11-22T18:56:09,198 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e2eea2cf5b56baf2f173c11708a11448/B of e2eea2cf5b56baf2f173c11708a11448 into 008449021d554a3d8ec2c6790c8c2a05(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:56:09,198 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e2eea2cf5b56baf2f173c11708a11448: 2024-11-22T18:56:09,198 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., storeName=e2eea2cf5b56baf2f173c11708a11448/B, priority=13, startTime=1732301768744; duration=0sec 2024-11-22T18:56:09,198 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:56:09,198 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e2eea2cf5b56baf2f173c11708a11448:B 2024-11-22T18:56:09,199 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T18:56:09,201 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T18:56:09,201 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1541): e2eea2cf5b56baf2f173c11708a11448/C is initiating minor compaction (all files) 2024-11-22T18:56:09,201 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e2eea2cf5b56baf2f173c11708a11448/C in TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:09,201 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/16e4272972eb4ead8fda36217b16b356, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/a6d069f36f9641eabee962b77fa8ad4b, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/2e4447bd01ba46b69667d63409b5848a] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp, totalSize=35.5 K 2024-11-22T18:56:09,202 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 16e4272972eb4ead8fda36217b16b356, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732301764332 2024-11-22T18:56:09,202 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting a6d069f36f9641eabee962b77fa8ad4b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732301765531 2024-11-22T18:56:09,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741985_1161 (size=31413) 2024-11-22T18:56:09,203 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e4447bd01ba46b69667d63409b5848a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732301766715 2024-11-22T18:56:09,210 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/4a4b68be641a4316a96dad951bab0a1d as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/4a4b68be641a4316a96dad951bab0a1d 2024-11-22T18:56:09,213 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e2eea2cf5b56baf2f173c11708a11448#C#compaction#135 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:56:09,213 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/C/97a0cbdee4ae4767a912c3f26e9ea79e is 50, key is test_row_0/C:col10/1732301766715/Put/seqid=0 2024-11-22T18:56:09,216 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e2eea2cf5b56baf2f173c11708a11448/A of e2eea2cf5b56baf2f173c11708a11448 into 4a4b68be641a4316a96dad951bab0a1d(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:56:09,216 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e2eea2cf5b56baf2f173c11708a11448: 2024-11-22T18:56:09,217 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., storeName=e2eea2cf5b56baf2f173c11708a11448/A, priority=13, startTime=1732301768744; duration=0sec 2024-11-22T18:56:09,217 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:56:09,217 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e2eea2cf5b56baf2f173c11708a11448:A 2024-11-22T18:56:09,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741986_1162 (size=12459) 2024-11-22T18:56:09,233 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/C/97a0cbdee4ae4767a912c3f26e9ea79e as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/97a0cbdee4ae4767a912c3f26e9ea79e 2024-11-22T18:56:09,241 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e2eea2cf5b56baf2f173c11708a11448/C of e2eea2cf5b56baf2f173c11708a11448 into 97a0cbdee4ae4767a912c3f26e9ea79e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:56:09,241 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e2eea2cf5b56baf2f173c11708a11448: 2024-11-22T18:56:09,241 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., storeName=e2eea2cf5b56baf2f173c11708a11448/C, priority=13, startTime=1732301768744; duration=0sec 2024-11-22T18:56:09,241 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:56:09,241 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e2eea2cf5b56baf2f173c11708a11448:C 2024-11-22T18:56:09,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:09,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43470 deadline: 1732301829386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:09,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:09,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:09,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:09,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:09,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:09,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43478 deadline: 1732301829386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:09,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43494 deadline: 1732301829386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:09,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:09,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:09,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:09,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:09,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:09,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:09,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:09,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43426 deadline: 1732301829396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:09,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:09,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43508 deadline: 1732301829396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:09,396 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:09,396 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:09,396 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:09,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:09,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:09,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:09,585 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=155, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/2852d5900da449429f6be52be2dc9a29 2024-11-22T18:56:09,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/B/fa9f7ad7fe79436183200ca58e6d185f is 50, key is test_row_0/B:col10/1732301767876/Put/seqid=0 2024-11-22T18:56:09,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741987_1163 (size=12151) 2024-11-22T18:56:09,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:09,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43470 deadline: 1732301829696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:09,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:09,697 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:09,697 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:09,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43494 deadline: 1732301829696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:09,697 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:09,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:09,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:09,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43478 deadline: 1732301829696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:09,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:09,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:09,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:09,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:09,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:09,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:09,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43426 deadline: 1732301829705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:09,709 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:09,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:09,709 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:09,709 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:09,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43508 deadline: 1732301829705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:09,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:09,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:09,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:09,998 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/B/fa9f7ad7fe79436183200ca58e6d185f 2024-11-22T18:56:10,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/C/60ccdfec1a594d74866ffbfbfc98f50a is 50, key is test_row_0/C:col10/1732301767876/Put/seqid=0 2024-11-22T18:56:10,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741988_1164 (size=12151) 2024-11-22T18:56:10,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-22T18:56:10,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:10,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43470 deadline: 1732301830205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:10,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:10,207 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:10,208 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:10,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43494 deadline: 1732301830205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:10,208 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:10,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:10,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43478 deadline: 1732301830205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:10,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:10,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:10,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:10,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:10,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:10,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:10,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:10,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43426 deadline: 1732301830216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:10,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:10,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43508 deadline: 1732301830216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:10,217 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:10,217 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:10,217 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:10,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:10,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:10,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:10,411 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/C/60ccdfec1a594d74866ffbfbfc98f50a 2024-11-22T18:56:10,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/2852d5900da449429f6be52be2dc9a29 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/2852d5900da449429f6be52be2dc9a29 2024-11-22T18:56:10,421 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/2852d5900da449429f6be52be2dc9a29, entries=150, sequenceid=155, filesize=30.4 K 2024-11-22T18:56:10,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/B/fa9f7ad7fe79436183200ca58e6d185f as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/fa9f7ad7fe79436183200ca58e6d185f 2024-11-22T18:56:10,427 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/fa9f7ad7fe79436183200ca58e6d185f, entries=150, sequenceid=155, filesize=11.9 K 2024-11-22T18:56:10,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/C/60ccdfec1a594d74866ffbfbfc98f50a as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/60ccdfec1a594d74866ffbfbfc98f50a 2024-11-22T18:56:10,432 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/60ccdfec1a594d74866ffbfbfc98f50a, entries=150, sequenceid=155, filesize=11.9 K 2024-11-22T18:56:10,433 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for e2eea2cf5b56baf2f173c11708a11448 in 1686ms, sequenceid=155, compaction requested=false 2024-11-22T18:56:10,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for e2eea2cf5b56baf2f173c11708a11448: 2024-11-22T18:56:10,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:10,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-22T18:56:10,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-22T18:56:10,437 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-22T18:56:10,437 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4560 sec 2024-11-22T18:56:10,439 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 2.4610 sec 2024-11-22T18:56:11,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:11,219 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T18:56:11,219 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e2eea2cf5b56baf2f173c11708a11448 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-22T18:56:11,228 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411223c0e205bd49248679dbea3a2354be0ca_e2eea2cf5b56baf2f173c11708a11448 is 50, key is test_row_0/A:col10/1732301769069/Put/seqid=0 2024-11-22T18:56:11,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741989_1165 (size=14794) 2024-11-22T18:56:11,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:11,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43494 deadline: 1732301831238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:11,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:11,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:11,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:11,246 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:11,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43426 deadline: 1732301831239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:11,246 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:11,247 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:11,247 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:11,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43508 deadline: 1732301831239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:11,247 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:11,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:11,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:11,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:11,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:11,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43470 deadline: 1732301831241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:11,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:11,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:11,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:11,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:11,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43478 deadline: 1732301831241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:11,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:11,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:11,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:11,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:11,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43494 deadline: 1732301831346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:11,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:11,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:11,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:11,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:11,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43426 deadline: 1732301831356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:11,358 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:11,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:11,358 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:11,358 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:11,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43508 deadline: 1732301831356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:11,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:11,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:11,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:11,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:11,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43478 deadline: 1732301831356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:11,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:11,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:11,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:11,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:11,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43470 deadline: 1732301831356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:11,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:11,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:11,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:11,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:11,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43494 deadline: 1732301831556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:11,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:11,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:11,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:11,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:11,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43426 deadline: 1732301831565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:11,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:11,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:11,568 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:11,568 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:11,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43508 deadline: 1732301831565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:11,568 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:11,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43478 deadline: 1732301831565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:11,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:11,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43470 deadline: 1732301831565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:11,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:11,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:11,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:11,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:11,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:11,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:11,569 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:11,569 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:11,569 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:11,638 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:56:11,644 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411223c0e205bd49248679dbea3a2354be0ca_e2eea2cf5b56baf2f173c11708a11448 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411223c0e205bd49248679dbea3a2354be0ca_e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:11,645 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/dc59fb3313594649abd5f526d3f8dd93, store: [table=TestAcidGuarantees family=A region=e2eea2cf5b56baf2f173c11708a11448] 2024-11-22T18:56:11,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/dc59fb3313594649abd5f526d3f8dd93 is 175, key is test_row_0/A:col10/1732301769069/Put/seqid=0 2024-11-22T18:56:11,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741990_1166 (size=39749) 2024-11-22T18:56:11,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:11,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43494 deadline: 1732301831865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:11,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:11,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:11,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:11,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:11,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43426 deadline: 1732301831876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:11,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:11,879 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:11,879 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:11,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43508 deadline: 1732301831876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:11,879 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:11,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:11,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:11,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43478 deadline: 1732301831876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:11,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:11,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:11,880 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:11,880 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:11,880 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:11,880 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:11,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43470 deadline: 1732301831876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:11,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:11,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:11,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:12,051 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=177, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/dc59fb3313594649abd5f526d3f8dd93 2024-11-22T18:56:12,060 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/B/d97eb7ee240b422bb348b651d40c93db is 50, key is test_row_0/B:col10/1732301769069/Put/seqid=0 2024-11-22T18:56:12,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741991_1167 (size=12151) 2024-11-22T18:56:12,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-22T18:56:12,155 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T18:56:12,156 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T18:56:12,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-22T18:56:12,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-22T18:56:12,158 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T18:56:12,159 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T18:56:12,159 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T18:56:12,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-22T18:56:12,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-22T18:56:12,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:12,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. as already flushing 2024-11-22T18:56:12,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:12,312 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:12,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:12,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:12,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:12,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43494 deadline: 1732301832386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:12,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:12,389 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:12,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43426 deadline: 1732301832386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:12,389 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:12,389 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:12,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:12,390 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:12,390 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:12,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43508 deadline: 1732301832386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:12,390 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:12,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:12,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:12,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:12,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43478 deadline: 1732301832386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:12,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:12,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:12,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:12,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:12,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:12,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43470 deadline: 1732301832386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:12,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:12,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:12,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:12,465 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/B/d97eb7ee240b422bb348b651d40c93db 2024-11-22T18:56:12,465 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-22T18:56:12,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:12,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. as already flushing 2024-11-22T18:56:12,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:12,465 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:12,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:12,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:12,474 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/C/76c09e1e95c143889744717feb96d876 is 50, key is test_row_0/C:col10/1732301769069/Put/seqid=0 2024-11-22T18:56:12,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-22T18:56:12,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741992_1168 (size=12151) 2024-11-22T18:56:12,617 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-22T18:56:12,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:12,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. as already flushing 2024-11-22T18:56:12,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:12,618 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:12,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:12,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:12,770 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-22T18:56:12,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:12,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. as already flushing 2024-11-22T18:56:12,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:12,771 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:12,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:12,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:12,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-22T18:56:12,879 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/C/76c09e1e95c143889744717feb96d876 2024-11-22T18:56:12,885 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/dc59fb3313594649abd5f526d3f8dd93 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/dc59fb3313594649abd5f526d3f8dd93 2024-11-22T18:56:12,890 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/dc59fb3313594649abd5f526d3f8dd93, entries=200, sequenceid=177, filesize=38.8 K 2024-11-22T18:56:12,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/B/d97eb7ee240b422bb348b651d40c93db as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/d97eb7ee240b422bb348b651d40c93db 2024-11-22T18:56:12,895 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/d97eb7ee240b422bb348b651d40c93db, entries=150, sequenceid=177, filesize=11.9 K 2024-11-22T18:56:12,896 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/C/76c09e1e95c143889744717feb96d876 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/76c09e1e95c143889744717feb96d876 2024-11-22T18:56:12,901 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/76c09e1e95c143889744717feb96d876, entries=150, sequenceid=177, filesize=11.9 K 2024-11-22T18:56:12,902 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=100.63 KB/103050 for e2eea2cf5b56baf2f173c11708a11448 in 1683ms, sequenceid=177, compaction requested=true 2024-11-22T18:56:12,902 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e2eea2cf5b56baf2f173c11708a11448: 2024-11-22T18:56:12,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e2eea2cf5b56baf2f173c11708a11448:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T18:56:12,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:56:12,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e2eea2cf5b56baf2f173c11708a11448:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T18:56:12,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:56:12,902 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T18:56:12,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e2eea2cf5b56baf2f173c11708a11448:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T18:56:12,902 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T18:56:12,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:56:12,903 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T18:56:12,904 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1541): e2eea2cf5b56baf2f173c11708a11448/B is initiating minor compaction (all files) 2024-11-22T18:56:12,904 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e2eea2cf5b56baf2f173c11708a11448/B in TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:12,904 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/008449021d554a3d8ec2c6790c8c2a05, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/fa9f7ad7fe79436183200ca58e6d185f, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/d97eb7ee240b422bb348b651d40c93db] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp, totalSize=35.9 K 2024-11-22T18:56:12,904 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102267 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T18:56:12,904 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1541): e2eea2cf5b56baf2f173c11708a11448/A is initiating minor compaction (all files) 2024-11-22T18:56:12,904 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e2eea2cf5b56baf2f173c11708a11448/A in TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:12,904 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/4a4b68be641a4316a96dad951bab0a1d, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/2852d5900da449429f6be52be2dc9a29, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/dc59fb3313594649abd5f526d3f8dd93] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp, totalSize=99.9 K 2024-11-22T18:56:12,904 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:12,904 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. files: [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/4a4b68be641a4316a96dad951bab0a1d, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/2852d5900da449429f6be52be2dc9a29, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/dc59fb3313594649abd5f526d3f8dd93] 2024-11-22T18:56:12,904 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 008449021d554a3d8ec2c6790c8c2a05, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732301766715 2024-11-22T18:56:12,905 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a4b68be641a4316a96dad951bab0a1d, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732301766715 2024-11-22T18:56:12,905 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting fa9f7ad7fe79436183200ca58e6d185f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732301767874 2024-11-22T18:56:12,905 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2852d5900da449429f6be52be2dc9a29, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732301767874 2024-11-22T18:56:12,905 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting d97eb7ee240b422bb348b651d40c93db, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1732301769024 2024-11-22T18:56:12,905 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc59fb3313594649abd5f526d3f8dd93, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1732301769024 2024-11-22T18:56:12,913 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e2eea2cf5b56baf2f173c11708a11448] 2024-11-22T18:56:12,916 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e2eea2cf5b56baf2f173c11708a11448#B#compaction#142 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:56:12,916 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/B/fc1eb86044524f07bba0c4d38443223f is 50, key is test_row_0/B:col10/1732301769069/Put/seqid=0 2024-11-22T18:56:12,918 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112239706a3efe404043bdedf0c4500942c8_e2eea2cf5b56baf2f173c11708a11448 store=[table=TestAcidGuarantees family=A region=e2eea2cf5b56baf2f173c11708a11448] 2024-11-22T18:56:12,920 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112239706a3efe404043bdedf0c4500942c8_e2eea2cf5b56baf2f173c11708a11448, store=[table=TestAcidGuarantees family=A region=e2eea2cf5b56baf2f173c11708a11448] 2024-11-22T18:56:12,921 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112239706a3efe404043bdedf0c4500942c8_e2eea2cf5b56baf2f173c11708a11448 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e2eea2cf5b56baf2f173c11708a11448] 2024-11-22T18:56:12,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-22T18:56:12,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:12,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741993_1169 (size=12561) 2024-11-22T18:56:12,924 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing e2eea2cf5b56baf2f173c11708a11448 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-22T18:56:12,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741994_1170 (size=4469) 2024-11-22T18:56:12,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122e5e8adb2eb704cc7a824887d99ca7dfa_e2eea2cf5b56baf2f173c11708a11448 is 50, key is test_row_0/A:col10/1732301771239/Put/seqid=0 2024-11-22T18:56:12,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741995_1171 (size=12304) 2024-11-22T18:56:13,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-22T18:56:13,329 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e2eea2cf5b56baf2f173c11708a11448#A#compaction#141 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:56:13,330 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/7fe7f94b49884417a715b6e194311664 is 175, key is test_row_0/A:col10/1732301769069/Put/seqid=0 2024-11-22T18:56:13,331 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/B/fc1eb86044524f07bba0c4d38443223f as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/fc1eb86044524f07bba0c4d38443223f 2024-11-22T18:56:13,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741996_1172 (size=31515) 2024-11-22T18:56:13,337 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e2eea2cf5b56baf2f173c11708a11448/B of e2eea2cf5b56baf2f173c11708a11448 into fc1eb86044524f07bba0c4d38443223f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:56:13,337 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e2eea2cf5b56baf2f173c11708a11448: 2024-11-22T18:56:13,337 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., storeName=e2eea2cf5b56baf2f173c11708a11448/B, priority=13, startTime=1732301772902; duration=0sec 2024-11-22T18:56:13,337 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:56:13,337 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e2eea2cf5b56baf2f173c11708a11448:B 2024-11-22T18:56:13,337 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T18:56:13,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:56:13,350 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T18:56:13,350 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1541): e2eea2cf5b56baf2f173c11708a11448/C is initiating minor compaction (all files) 2024-11-22T18:56:13,350 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e2eea2cf5b56baf2f173c11708a11448/C in TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:13,350 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/97a0cbdee4ae4767a912c3f26e9ea79e, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/60ccdfec1a594d74866ffbfbfc98f50a, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/76c09e1e95c143889744717feb96d876] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp, totalSize=35.9 K 2024-11-22T18:56:13,351 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 97a0cbdee4ae4767a912c3f26e9ea79e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732301766715 2024-11-22T18:56:13,351 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 60ccdfec1a594d74866ffbfbfc98f50a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732301767874 2024-11-22T18:56:13,352 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 76c09e1e95c143889744717feb96d876, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1732301769024 2024-11-22T18:56:13,354 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122e5e8adb2eb704cc7a824887d99ca7dfa_e2eea2cf5b56baf2f173c11708a11448 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122e5e8adb2eb704cc7a824887d99ca7dfa_e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:13,357 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/7fe7f94b49884417a715b6e194311664 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/7fe7f94b49884417a715b6e194311664 2024-11-22T18:56:13,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/420a9f37d1714f67bb72e2a5c3b88abc, store: [table=TestAcidGuarantees family=A region=e2eea2cf5b56baf2f173c11708a11448] 2024-11-22T18:56:13,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/420a9f37d1714f67bb72e2a5c3b88abc is 175, key is test_row_0/A:col10/1732301771239/Put/seqid=0 2024-11-22T18:56:13,364 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e2eea2cf5b56baf2f173c11708a11448/A of e2eea2cf5b56baf2f173c11708a11448 into 7fe7f94b49884417a715b6e194311664(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:56:13,364 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e2eea2cf5b56baf2f173c11708a11448: 2024-11-22T18:56:13,364 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., storeName=e2eea2cf5b56baf2f173c11708a11448/A, priority=13, startTime=1732301772902; duration=0sec 2024-11-22T18:56:13,364 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:56:13,364 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e2eea2cf5b56baf2f173c11708a11448:A 2024-11-22T18:56:13,376 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e2eea2cf5b56baf2f173c11708a11448#C#compaction#144 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:56:13,377 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/C/91c8c4cf5aae45a98b3830dd59046350 is 50, key is test_row_0/C:col10/1732301769069/Put/seqid=0 2024-11-22T18:56:13,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741997_1173 (size=31105) 2024-11-22T18:56:13,380 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=195, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/420a9f37d1714f67bb72e2a5c3b88abc 2024-11-22T18:56:13,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741998_1174 (size=12561) 2024-11-22T18:56:13,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/B/613b1524118e45b8a08f062ba49275b3 is 50, key is test_row_0/B:col10/1732301771239/Put/seqid=0 2024-11-22T18:56:13,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:13,397 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. as already flushing 2024-11-22T18:56:13,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741999_1175 (size=12151) 2024-11-22T18:56:13,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:13,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43508 deadline: 1732301833416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:13,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:13,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:13,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:13,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:13,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43478 deadline: 1732301833417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:13,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:13,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43494 deadline: 1732301833419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:13,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:13,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:13,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:13,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:13,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:13,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43426 deadline: 1732301833419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:13,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:13,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:13,423 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:13,423 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:13,423 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:13,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:13,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43470 deadline: 1732301833420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:13,424 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:13,424 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:13,424 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:13,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:13,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43508 deadline: 1732301833526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:13,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:13,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:13,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:13,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43478 deadline: 1732301833526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:13,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:13,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:13,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:13,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43494 deadline: 1732301833526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:13,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:13,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:13,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:13,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43426 deadline: 1732301833526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:13,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:13,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:13,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:13,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:13,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43470 deadline: 1732301833526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:13,528 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:13,528 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:13,528 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:13,528 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:13,528 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:13,528 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:13,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:13,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43508 deadline: 1732301833736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:13,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:13,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:13,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:13,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:13,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43478 deadline: 1732301833736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:13,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:13,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:13,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:13,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:13,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43494 deadline: 1732301833736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:13,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:13,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:13,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:13,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:13,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43426 deadline: 1732301833736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:13,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:13,741 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:13,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43470 deadline: 1732301833736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:13,741 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:13,741 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:13,741 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:13,741 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:13,741 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:13,791 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/C/91c8c4cf5aae45a98b3830dd59046350 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/91c8c4cf5aae45a98b3830dd59046350 2024-11-22T18:56:13,796 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e2eea2cf5b56baf2f173c11708a11448/C of e2eea2cf5b56baf2f173c11708a11448 into 91c8c4cf5aae45a98b3830dd59046350(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:56:13,796 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e2eea2cf5b56baf2f173c11708a11448: 2024-11-22T18:56:13,796 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., storeName=e2eea2cf5b56baf2f173c11708a11448/C, priority=13, startTime=1732301772902; duration=0sec 2024-11-22T18:56:13,796 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:56:13,796 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e2eea2cf5b56baf2f173c11708a11448:C 2024-11-22T18:56:13,800 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/B/613b1524118e45b8a08f062ba49275b3 2024-11-22T18:56:13,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/C/f28de847f7a648a7970a141330250056 is 50, key is test_row_0/C:col10/1732301771239/Put/seqid=0 2024-11-22T18:56:13,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742000_1176 (size=12151) 2024-11-22T18:56:14,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:14,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43508 deadline: 1732301834045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:14,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:14,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:14,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43478 deadline: 1732301834045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:14,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:14,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:14,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:14,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:14,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:14,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43494 deadline: 1732301834045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:14,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:14,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:14,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:14,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:14,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:14,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43426 deadline: 1732301834045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:14,048 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:14,048 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:14,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:14,048 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:14,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43470 deadline: 1732301834046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:14,049 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:14,049 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:14,049 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:14,232 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/C/f28de847f7a648a7970a141330250056 2024-11-22T18:56:14,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/420a9f37d1714f67bb72e2a5c3b88abc as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/420a9f37d1714f67bb72e2a5c3b88abc 2024-11-22T18:56:14,243 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/420a9f37d1714f67bb72e2a5c3b88abc, entries=150, sequenceid=195, filesize=30.4 K 2024-11-22T18:56:14,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/B/613b1524118e45b8a08f062ba49275b3 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/613b1524118e45b8a08f062ba49275b3 2024-11-22T18:56:14,249 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/613b1524118e45b8a08f062ba49275b3, entries=150, sequenceid=195, filesize=11.9 K 2024-11-22T18:56:14,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/C/f28de847f7a648a7970a141330250056 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/f28de847f7a648a7970a141330250056 2024-11-22T18:56:14,255 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/f28de847f7a648a7970a141330250056, entries=150, sequenceid=195, filesize=11.9 K 2024-11-22T18:56:14,256 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for e2eea2cf5b56baf2f173c11708a11448 in 1332ms, sequenceid=195, compaction requested=false 2024-11-22T18:56:14,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for e2eea2cf5b56baf2f173c11708a11448: 2024-11-22T18:56:14,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:14,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-22T18:56:14,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-22T18:56:14,259 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-22T18:56:14,259 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0980 sec 2024-11-22T18:56:14,261 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 2.1040 sec 2024-11-22T18:56:14,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-22T18:56:14,306 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T18:56:14,306 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T18:56:14,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-22T18:56:14,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-22T18:56:14,309 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T18:56:14,309 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T18:56:14,310 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T18:56:14,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-22T18:56:14,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-22T18:56:14,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:14,462 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing e2eea2cf5b56baf2f173c11708a11448 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-22T18:56:14,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122a10eafd0c8664593a2dd7b544c7f6df7_e2eea2cf5b56baf2f173c11708a11448 is 50, key is test_row_0/A:col10/1732301773417/Put/seqid=0 2024-11-22T18:56:14,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742001_1177 (size=12304) 2024-11-22T18:56:14,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:14,558 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. as already flushing 2024-11-22T18:56:14,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:14,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43478 deadline: 1732301834575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:14,580 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:14,580 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:14,580 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:14,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:14,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43470 deadline: 1732301834577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:14,582 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:14,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:14,582 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:14,582 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:14,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43426 deadline: 1732301834578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:14,582 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:14,582 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:14,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:14,582 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:14,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43494 deadline: 1732301834579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:14,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:14,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:14,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:14,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:14,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43508 deadline: 1732301834580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:14,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:14,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:14,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:14,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-22T18:56:14,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:14,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43478 deadline: 1732301834685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:14,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:14,687 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:14,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43470 deadline: 1732301834685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:14,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:14,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:14,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:14,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:14,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:14,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:14,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43426 deadline: 1732301834685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:14,688 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:14,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:14,688 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:14,688 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:14,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43494 deadline: 1732301834686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:14,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:14,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:14,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:14,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:14,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43508 deadline: 1732301834686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:14,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:14,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:14,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:14,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:56:14,887 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122a10eafd0c8664593a2dd7b544c7f6df7_e2eea2cf5b56baf2f173c11708a11448 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122a10eafd0c8664593a2dd7b544c7f6df7_e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:14,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/4eecacf48a5d45029a48814dbd557b78, store: [table=TestAcidGuarantees family=A region=e2eea2cf5b56baf2f173c11708a11448] 2024-11-22T18:56:14,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/4eecacf48a5d45029a48814dbd557b78 is 175, key is test_row_0/A:col10/1732301773417/Put/seqid=0 2024-11-22T18:56:14,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742002_1178 (size=31105) 2024-11-22T18:56:14,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:14,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43478 deadline: 1732301834895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:14,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:14,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:14,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:14,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:14,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43470 deadline: 1732301834895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:14,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:14,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:14,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:14,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:14,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43426 deadline: 1732301834896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:14,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:14,899 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:14,899 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:14,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43508 deadline: 1732301834896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:14,899 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:14,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:14,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:14,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:14,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:14,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43494 deadline: 1732301834896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:14,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:14,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:14,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:14,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-22T18:56:15,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:15,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43478 deadline: 1732301835205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:15,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:15,207 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:15,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43470 deadline: 1732301835205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:15,207 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:15,207 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:15,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:15,207 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:15,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43426 deadline: 1732301835205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:15,207 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:15,207 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:15,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:15,208 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:15,208 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:15,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43508 deadline: 1732301835206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:15,208 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:15,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:15,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43494 deadline: 1732301835206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:15,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:15,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:15,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:15,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:15,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:15,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:15,293 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=216, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/4eecacf48a5d45029a48814dbd557b78 2024-11-22T18:56:15,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/B/b59bb2f5872c42e8b929fd704da99738 is 50, key is test_row_0/B:col10/1732301773417/Put/seqid=0 2024-11-22T18:56:15,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742003_1179 (size=12151) 2024-11-22T18:56:15,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-22T18:56:15,706 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/B/b59bb2f5872c42e8b929fd704da99738 2024-11-22T18:56:15,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/C/87f173447ece4f9da5aab197dc9af1a9 is 50, key is test_row_0/C:col10/1732301773417/Put/seqid=0 2024-11-22T18:56:15,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742004_1180 (size=12151) 2024-11-22T18:56:15,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:15,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43470 deadline: 1732301835716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:15,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:15,719 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:15,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43478 deadline: 1732301835716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:15,719 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:15,719 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:15,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:15,719 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:15,719 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:15,719 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:15,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43426 deadline: 1732301835716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:15,719 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:15,719 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:15,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:15,720 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:15,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43494 deadline: 1732301835716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:15,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:15,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43508 deadline: 1732301835716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:15,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:15,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:15,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:15,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:15,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:15,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:16,119 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/C/87f173447ece4f9da5aab197dc9af1a9 2024-11-22T18:56:16,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/4eecacf48a5d45029a48814dbd557b78 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/4eecacf48a5d45029a48814dbd557b78 2024-11-22T18:56:16,143 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/4eecacf48a5d45029a48814dbd557b78, entries=150, sequenceid=216, filesize=30.4 K 2024-11-22T18:56:16,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/B/b59bb2f5872c42e8b929fd704da99738 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/b59bb2f5872c42e8b929fd704da99738 2024-11-22T18:56:16,154 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/b59bb2f5872c42e8b929fd704da99738, entries=150, sequenceid=216, filesize=11.9 K 2024-11-22T18:56:16,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/C/87f173447ece4f9da5aab197dc9af1a9 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/87f173447ece4f9da5aab197dc9af1a9 2024-11-22T18:56:16,159 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/87f173447ece4f9da5aab197dc9af1a9, entries=150, sequenceid=216, filesize=11.9 K 2024-11-22T18:56:16,160 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=107.34 KB/109920 for e2eea2cf5b56baf2f173c11708a11448 in 1698ms, sequenceid=216, compaction requested=true 2024-11-22T18:56:16,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for e2eea2cf5b56baf2f173c11708a11448: 2024-11-22T18:56:16,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:16,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-22T18:56:16,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-22T18:56:16,163 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-22T18:56:16,163 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8520 sec 2024-11-22T18:56:16,166 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 1.8590 sec 2024-11-22T18:56:16,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-22T18:56:16,456 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T18:56:16,457 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T18:56:16,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-11-22T18:56:16,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-22T18:56:16,458 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T18:56:16,459 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T18:56:16,459 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T18:56:16,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-22T18:56:16,614 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-22T18:56:16,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:16,614 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing e2eea2cf5b56baf2f173c11708a11448 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-22T18:56:16,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411220d0dc1af500b4eeda891bb17c1d12e76_e2eea2cf5b56baf2f173c11708a11448 is 50, key is test_row_0/A:col10/1732301774575/Put/seqid=0 2024-11-22T18:56:16,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742005_1181 (size=12304) 2024-11-22T18:56:16,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:16,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. as already flushing 2024-11-22T18:56:16,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:16,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:16,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43508 deadline: 1732301836743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:16,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43478 deadline: 1732301836744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:16,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:16,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43494 deadline: 1732301836746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:16,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:16,750 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:16,750 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:16,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:16,750 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:16,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:16,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:16,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:16,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:16,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:16,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43426 deadline: 1732301836750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:16,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:16,755 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:16,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43470 deadline: 1732301836750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:16,755 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:16,755 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:16,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:16,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:16,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:16,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-22T18:56:16,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:16,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:16,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43508 deadline: 1732301836856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:16,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43478 deadline: 1732301836856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:16,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:16,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43494 deadline: 1732301836856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:16,858 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:16,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:16,858 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:16,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:16,858 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:16,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:16,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:16,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:16,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:16,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:16,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43426 deadline: 1732301836866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:16,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:16,870 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:16,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43470 deadline: 1732301836866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:16,870 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:16,870 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:16,870 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:16,870 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:16,870 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:17,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:56:17,032 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411220d0dc1af500b4eeda891bb17c1d12e76_e2eea2cf5b56baf2f173c11708a11448 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411220d0dc1af500b4eeda891bb17c1d12e76_e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:17,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/cb5507c6c6164ebca1f8442c94e6ef50, store: [table=TestAcidGuarantees family=A region=e2eea2cf5b56baf2f173c11708a11448] 2024-11-22T18:56:17,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/cb5507c6c6164ebca1f8442c94e6ef50 is 175, key is test_row_0/A:col10/1732301774575/Put/seqid=0 2024-11-22T18:56:17,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742006_1182 (size=31105) 2024-11-22T18:56:17,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:17,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43478 deadline: 1732301837066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:17,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:17,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43508 deadline: 1732301837066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:17,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:17,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:17,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:17,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:17,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43494 deadline: 1732301837066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:17,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:17,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:17,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:17,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:17,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:17,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:17,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:17,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43426 deadline: 1732301837076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:17,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:17,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43470 deadline: 1732301837076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:17,079 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:17,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:17,079 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:17,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:17,079 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:17,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:17,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-22T18:56:17,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:17,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:17,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43478 deadline: 1732301837375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:17,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43508 deadline: 1732301837375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:17,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:17,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43494 deadline: 1732301837375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:17,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:17,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:17,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:17,379 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:17,379 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:17,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:17,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:17,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:17,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:17,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:17,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43426 deadline: 1732301837386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:17,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:17,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43470 deadline: 1732301837386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:17,391 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:17,391 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:17,391 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:17,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:17,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:17,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:17,440 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=235, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/cb5507c6c6164ebca1f8442c94e6ef50 2024-11-22T18:56:17,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/B/91c76528f20e485982bb5dcf93d5c0f3 is 50, key is test_row_0/B:col10/1732301774575/Put/seqid=0 2024-11-22T18:56:17,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742007_1183 (size=12151) 2024-11-22T18:56:17,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-22T18:56:17,854 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/B/91c76528f20e485982bb5dcf93d5c0f3 2024-11-22T18:56:17,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/C/a4441327641a432187b040741a155605 is 50, key is test_row_0/C:col10/1732301774575/Put/seqid=0 2024-11-22T18:56:17,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742008_1184 (size=12151) 2024-11-22T18:56:17,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:17,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43478 deadline: 1732301837886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:17,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:17,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:17,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:17,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:17,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43508 deadline: 1732301837886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:17,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:17,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:17,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43494 deadline: 1732301837886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:17,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:17,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:17,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:17,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:17,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:17,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:17,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43426 deadline: 1732301837895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:17,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:17,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43470 deadline: 1732301837895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:17,896 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:17,896 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:17,896 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:17,896 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:17,896 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:17,896 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:18,269 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/C/a4441327641a432187b040741a155605 2024-11-22T18:56:18,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/cb5507c6c6164ebca1f8442c94e6ef50 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/cb5507c6c6164ebca1f8442c94e6ef50 2024-11-22T18:56:18,278 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/cb5507c6c6164ebca1f8442c94e6ef50, entries=150, sequenceid=235, filesize=30.4 K 2024-11-22T18:56:18,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/B/91c76528f20e485982bb5dcf93d5c0f3 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/91c76528f20e485982bb5dcf93d5c0f3 2024-11-22T18:56:18,288 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/91c76528f20e485982bb5dcf93d5c0f3, entries=150, sequenceid=235, filesize=11.9 K 2024-11-22T18:56:18,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/C/a4441327641a432187b040741a155605 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/a4441327641a432187b040741a155605 2024-11-22T18:56:18,293 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/a4441327641a432187b040741a155605, entries=150, sequenceid=235, filesize=11.9 K 2024-11-22T18:56:18,294 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=100.63 KB/103050 for e2eea2cf5b56baf2f173c11708a11448 in 1680ms, sequenceid=235, compaction requested=true 2024-11-22T18:56:18,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for e2eea2cf5b56baf2f173c11708a11448: 2024-11-22T18:56:18,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:18,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-22T18:56:18,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-22T18:56:18,297 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-22T18:56:18,297 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8360 sec 2024-11-22T18:56:18,299 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 1.8410 sec 2024-11-22T18:56:18,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-22T18:56:18,605 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T18:56:18,606 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T18:56:18,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-11-22T18:56:18,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-22T18:56:18,608 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T18:56:18,609 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T18:56:18,609 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T18:56:18,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-22T18:56:18,761 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-22T18:56:18,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:18,762 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing e2eea2cf5b56baf2f173c11708a11448 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-22T18:56:18,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122c489caf40f1b42a5aff4675f81ae9230_e2eea2cf5b56baf2f173c11708a11448 is 50, key is test_row_0/A:col10/1732301776748/Put/seqid=0 2024-11-22T18:56:18,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742009_1185 (size=12304) 2024-11-22T18:56:18,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:18,898 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. as already flushing 2024-11-22T18:56:18,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:18,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43478 deadline: 1732301838916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:18,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:18,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:18,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:18,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43494 deadline: 1732301838917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:18,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:18,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:18,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:18,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:18,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:18,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43426 deadline: 1732301838917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:18,924 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:18,924 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:18,924 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:18,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-22T18:56:18,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:18,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43470 deadline: 1732301838921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:18,928 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:18,928 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:18,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:18,928 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:18,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43508 deadline: 1732301838923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:18,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:18,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:18,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:19,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:19,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43478 deadline: 1732301839025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:19,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:19,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:19,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:19,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:19,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43494 deadline: 1732301839025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:19,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:19,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:19,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:19,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:19,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43426 deadline: 1732301839025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:19,028 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:19,028 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:19,028 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:19,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:19,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43470 deadline: 1732301839036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:19,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:19,042 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:19,042 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:19,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43508 deadline: 1732301839036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:19,042 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:19,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:19,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:19,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:19,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:56:19,184 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122c489caf40f1b42a5aff4675f81ae9230_e2eea2cf5b56baf2f173c11708a11448 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122c489caf40f1b42a5aff4675f81ae9230_e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:19,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/ebc01b2f497b4d12bc7ee03111a98b59, store: [table=TestAcidGuarantees family=A region=e2eea2cf5b56baf2f173c11708a11448] 2024-11-22T18:56:19,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/ebc01b2f497b4d12bc7ee03111a98b59 is 175, key is test_row_0/A:col10/1732301776748/Put/seqid=0 2024-11-22T18:56:19,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742010_1186 (size=31105) 2024-11-22T18:56:19,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-22T18:56:19,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:19,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43478 deadline: 1732301839236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:19,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:19,240 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:19,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43494 deadline: 1732301839236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:19,240 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:19,240 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:19,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:19,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:19,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:19,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43426 deadline: 1732301839236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:19,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:19,240 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:19,240 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:19,240 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:19,249 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:19,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43470 deadline: 1732301839245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:19,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:19,250 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:19,250 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:19,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43508 deadline: 1732301839245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:19,250 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:19,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:19,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:19,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:19,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:19,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43478 deadline: 1732301839545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:19,549 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:19,549 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:19,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:19,549 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:19,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43494 deadline: 1732301839545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:19,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:19,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:19,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:19,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:19,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43426 deadline: 1732301839545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:19,550 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:19,550 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:19,550 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:19,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:19,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43470 deadline: 1732301839556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:19,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:19,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:19,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:19,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:19,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43508 deadline: 1732301839556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:19,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:19,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:19,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:19,591 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=253, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/ebc01b2f497b4d12bc7ee03111a98b59 2024-11-22T18:56:19,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/B/3d334b3d924c4390a8984aa301c2bdcc is 50, key is test_row_0/B:col10/1732301776748/Put/seqid=0 2024-11-22T18:56:19,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742011_1187 (size=12151) 2024-11-22T18:56:19,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-22T18:56:20,007 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/B/3d334b3d924c4390a8984aa301c2bdcc 2024-11-22T18:56:20,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/C/98b88b2c22464feb92d6f610849dfab7 is 50, key is test_row_0/C:col10/1732301776748/Put/seqid=0 2024-11-22T18:56:20,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742012_1188 (size=12151) 2024-11-22T18:56:20,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:20,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43494 deadline: 1732301840055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:20,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:20,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:20,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:20,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:20,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:20,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43478 deadline: 1732301840055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:20,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43426 deadline: 1732301840056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:20,061 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:20,061 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:20,061 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:20,061 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:20,061 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:20,061 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:20,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:20,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43470 deadline: 1732301840066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:20,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:20,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:20,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:20,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:20,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43508 deadline: 1732301840066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:20,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:20,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:20,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:20,421 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/C/98b88b2c22464feb92d6f610849dfab7 2024-11-22T18:56:20,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/ebc01b2f497b4d12bc7ee03111a98b59 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/ebc01b2f497b4d12bc7ee03111a98b59 2024-11-22T18:56:20,431 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/ebc01b2f497b4d12bc7ee03111a98b59, entries=150, sequenceid=253, filesize=30.4 K 2024-11-22T18:56:20,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/B/3d334b3d924c4390a8984aa301c2bdcc as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/3d334b3d924c4390a8984aa301c2bdcc 2024-11-22T18:56:20,436 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/3d334b3d924c4390a8984aa301c2bdcc, entries=150, sequenceid=253, filesize=11.9 K 2024-11-22T18:56:20,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/C/98b88b2c22464feb92d6f610849dfab7 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/98b88b2c22464feb92d6f610849dfab7 2024-11-22T18:56:20,441 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/98b88b2c22464feb92d6f610849dfab7, entries=150, sequenceid=253, filesize=11.9 K 2024-11-22T18:56:20,442 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for e2eea2cf5b56baf2f173c11708a11448 in 1680ms, sequenceid=253, compaction requested=true 2024-11-22T18:56:20,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for e2eea2cf5b56baf2f173c11708a11448: 2024-11-22T18:56:20,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:20,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-22T18:56:20,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-22T18:56:20,446 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-22T18:56:20,446 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8350 sec 2024-11-22T18:56:20,448 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 1.8410 sec 2024-11-22T18:56:20,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-22T18:56:20,756 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T18:56:20,756 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T18:56:20,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-11-22T18:56:20,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-22T18:56:20,758 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T18:56:20,759 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T18:56:20,759 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T18:56:20,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-22T18:56:20,911 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-22T18:56:20,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:20,912 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing e2eea2cf5b56baf2f173c11708a11448 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-22T18:56:20,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411222406b56a48c244abbc5ac1bff479fa36_e2eea2cf5b56baf2f173c11708a11448 is 50, key is test_row_0/A:col10/1732301778921/Put/seqid=0 2024-11-22T18:56:20,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742013_1189 (size=12454) 2024-11-22T18:56:20,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:56:20,929 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411222406b56a48c244abbc5ac1bff479fa36_e2eea2cf5b56baf2f173c11708a11448 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411222406b56a48c244abbc5ac1bff479fa36_e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:20,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/f1a786746f2745d881c1d0dac9ccf1b1, store: [table=TestAcidGuarantees family=A region=e2eea2cf5b56baf2f173c11708a11448] 2024-11-22T18:56:20,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/f1a786746f2745d881c1d0dac9ccf1b1 is 175, key is test_row_0/A:col10/1732301778921/Put/seqid=0 2024-11-22T18:56:20,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742014_1190 (size=31255) 2024-11-22T18:56:21,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:21,067 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. as already flushing 2024-11-22T18:56:21,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-22T18:56:21,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:21,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43478 deadline: 1732301841082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:21,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:21,091 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:21,091 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:21,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43494 deadline: 1732301841085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:21,091 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:21,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:21,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:21,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:21,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:21,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43470 deadline: 1732301841086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:21,091 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:21,091 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:21,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:21,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:21,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43508 deadline: 1732301841089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:21,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:21,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:21,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:21,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:21,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43426 deadline: 1732301841090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:21,096 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:21,096 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:21,096 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:21,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:21,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43478 deadline: 1732301841196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:21,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:21,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:21,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43494 deadline: 1732301841196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:21,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:21,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:21,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:21,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:21,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43470 deadline: 1732301841196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:21,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:21,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:21,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:21,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:21,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:21,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:21,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43508 deadline: 1732301841205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:21,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:21,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:21,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:21,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:21,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43426 deadline: 1732301841205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:21,206 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:21,206 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:21,207 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:21,344 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=271, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/f1a786746f2745d881c1d0dac9ccf1b1 2024-11-22T18:56:21,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/B/95549494c9264735b083aa17ca33bf27 is 50, key is test_row_0/B:col10/1732301778921/Put/seqid=0 2024-11-22T18:56:21,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742015_1191 (size=12301) 2024-11-22T18:56:21,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-22T18:56:21,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:21,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43478 deadline: 1732301841405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:21,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:21,413 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:21,413 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:21,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43494 deadline: 1732301841405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:21,413 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:21,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:21,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:21,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:21,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:21,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43470 deadline: 1732301841405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:21,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:21,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:21,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:21,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:21,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43508 deadline: 1732301841415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:21,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:21,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:21,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:21,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:21,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43426 deadline: 1732301841415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:21,420 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:21,420 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:21,420 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:21,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:21,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43478 deadline: 1732301841726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:21,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:21,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:21,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:21,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43494 deadline: 1732301841726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:21,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:21,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:21,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:21,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43470 deadline: 1732301841726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:21,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:21,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:21,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:21,731 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:21,732 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:21,732 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:21,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43508 deadline: 1732301841726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:21,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:21,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:21,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:21,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:21,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43426 deadline: 1732301841726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:21,732 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:21,732 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:21,732 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:21,759 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/B/95549494c9264735b083aa17ca33bf27 2024-11-22T18:56:21,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/C/2a791ea2f7914ae4b31f31a2287e2580 is 50, key is test_row_0/C:col10/1732301778921/Put/seqid=0 2024-11-22T18:56:21,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742016_1192 (size=12301) 2024-11-22T18:56:21,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-22T18:56:22,172 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/C/2a791ea2f7914ae4b31f31a2287e2580 2024-11-22T18:56:22,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/f1a786746f2745d881c1d0dac9ccf1b1 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/f1a786746f2745d881c1d0dac9ccf1b1 2024-11-22T18:56:22,182 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/f1a786746f2745d881c1d0dac9ccf1b1, entries=150, sequenceid=271, filesize=30.5 K 2024-11-22T18:56:22,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/B/95549494c9264735b083aa17ca33bf27 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/95549494c9264735b083aa17ca33bf27 2024-11-22T18:56:22,186 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/95549494c9264735b083aa17ca33bf27, entries=150, sequenceid=271, filesize=12.0 K 2024-11-22T18:56:22,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/C/2a791ea2f7914ae4b31f31a2287e2580 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/2a791ea2f7914ae4b31f31a2287e2580 2024-11-22T18:56:22,192 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/2a791ea2f7914ae4b31f31a2287e2580, entries=150, sequenceid=271, filesize=12.0 K 2024-11-22T18:56:22,193 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for e2eea2cf5b56baf2f173c11708a11448 in 1282ms, sequenceid=271, compaction requested=true 2024-11-22T18:56:22,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for e2eea2cf5b56baf2f173c11708a11448: 2024-11-22T18:56:22,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:22,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-11-22T18:56:22,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-11-22T18:56:22,196 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-22T18:56:22,196 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4360 sec 2024-11-22T18:56:22,198 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 1.4410 sec 2024-11-22T18:56:22,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:22,240 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T18:56:22,240 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e2eea2cf5b56baf2f173c11708a11448 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-22T18:56:22,249 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411222d9addab35244f24b434ff3a858fb96a_e2eea2cf5b56baf2f173c11708a11448 is 50, key is test_row_0/A:col10/1732301782239/Put/seqid=0 2024-11-22T18:56:22,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742017_1193 (size=17534) 2024-11-22T18:56:22,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:22,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43494 deadline: 1732301842256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:22,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:22,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:22,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:22,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:22,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43508 deadline: 1732301842257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:22,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:22,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:22,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:22,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:22,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43426 deadline: 1732301842258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:22,267 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:22,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:22,267 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:22,267 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:22,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43470 deadline: 1732301842259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:22,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:22,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:22,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:22,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:22,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43478 deadline: 1732301842259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:22,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:22,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:22,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:22,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:22,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43494 deadline: 1732301842365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:22,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:22,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:22,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:22,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:22,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43508 deadline: 1732301842376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:22,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:22,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:22,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:22,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:22,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43426 deadline: 1732301842376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:22,381 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:22,381 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:22,381 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:22,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:22,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43470 deadline: 1732301842376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:22,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:22,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:22,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:22,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:22,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43478 deadline: 1732301842376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:22,382 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:22,382 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:22,382 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:22,428 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-22T18:56:22,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:22,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43494 deadline: 1732301842575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:22,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:22,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:22,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:22,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:22,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43508 deadline: 1732301842586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:22,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:22,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:22,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43426 deadline: 1732301842586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:22,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:22,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:22,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:22,591 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:22,591 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:22,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43470 deadline: 1732301842586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:22,591 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:22,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:22,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:22,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:22,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:22,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43478 deadline: 1732301842586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:22,592 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:22,592 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:22,592 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:22,655 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:56:22,660 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411222d9addab35244f24b434ff3a858fb96a_e2eea2cf5b56baf2f173c11708a11448 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411222d9addab35244f24b434ff3a858fb96a_e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:22,662 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/fc7eb3f7a996421f862800407cee6937, store: [table=TestAcidGuarantees family=A region=e2eea2cf5b56baf2f173c11708a11448] 2024-11-22T18:56:22,662 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/fc7eb3f7a996421f862800407cee6937 is 175, key is test_row_0/A:col10/1732301782239/Put/seqid=0 2024-11-22T18:56:22,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742018_1194 (size=48639) 2024-11-22T18:56:22,668 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=291, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/fc7eb3f7a996421f862800407cee6937 2024-11-22T18:56:22,678 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/B/95e286c197ee41298f221130f3a58072 is 50, key is test_row_0/B:col10/1732301782239/Put/seqid=0 2024-11-22T18:56:22,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742019_1195 (size=12301) 2024-11-22T18:56:22,685 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/B/95e286c197ee41298f221130f3a58072 2024-11-22T18:56:22,694 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/C/33b729e14c094aa9b3089c48997f20c2 is 50, key is test_row_0/C:col10/1732301782239/Put/seqid=0 2024-11-22T18:56:22,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742020_1196 (size=12301) 2024-11-22T18:56:22,701 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/C/33b729e14c094aa9b3089c48997f20c2 2024-11-22T18:56:22,707 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/fc7eb3f7a996421f862800407cee6937 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/fc7eb3f7a996421f862800407cee6937 2024-11-22T18:56:22,712 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/fc7eb3f7a996421f862800407cee6937, entries=250, sequenceid=291, filesize=47.5 K 2024-11-22T18:56:22,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/B/95e286c197ee41298f221130f3a58072 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/95e286c197ee41298f221130f3a58072 2024-11-22T18:56:22,718 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/95e286c197ee41298f221130f3a58072, entries=150, sequenceid=291, filesize=12.0 K 2024-11-22T18:56:22,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/C/33b729e14c094aa9b3089c48997f20c2 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/33b729e14c094aa9b3089c48997f20c2 2024-11-22T18:56:22,726 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/33b729e14c094aa9b3089c48997f20c2, entries=150, sequenceid=291, filesize=12.0 K 2024-11-22T18:56:22,727 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for e2eea2cf5b56baf2f173c11708a11448 in 487ms, sequenceid=291, compaction requested=true 2024-11-22T18:56:22,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e2eea2cf5b56baf2f173c11708a11448: 2024-11-22T18:56:22,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e2eea2cf5b56baf2f173c11708a11448:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T18:56:22,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:56:22,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e2eea2cf5b56baf2f173c11708a11448:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T18:56:22,728 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 0 compacting, 7 eligible, 16 blocking 2024-11-22T18:56:22,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:56:22,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e2eea2cf5b56baf2f173c11708a11448:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T18:56:22,728 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 0 compacting, 7 eligible, 16 blocking 2024-11-22T18:56:22,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:56:22,731 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 7 files of size 235829 starting at candidate #0 after considering 15 permutations with 15 in ratio 2024-11-22T18:56:22,731 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 7 files of size 85767 starting at candidate #0 after considering 15 permutations with 15 in ratio 2024-11-22T18:56:22,731 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1541): e2eea2cf5b56baf2f173c11708a11448/A is initiating minor compaction (all files) 2024-11-22T18:56:22,731 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1541): e2eea2cf5b56baf2f173c11708a11448/B is initiating minor compaction (all files) 2024-11-22T18:56:22,731 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e2eea2cf5b56baf2f173c11708a11448/A in TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:22,731 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e2eea2cf5b56baf2f173c11708a11448/B in TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:22,731 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/7fe7f94b49884417a715b6e194311664, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/420a9f37d1714f67bb72e2a5c3b88abc, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/4eecacf48a5d45029a48814dbd557b78, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/cb5507c6c6164ebca1f8442c94e6ef50, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/ebc01b2f497b4d12bc7ee03111a98b59, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/f1a786746f2745d881c1d0dac9ccf1b1, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/fc7eb3f7a996421f862800407cee6937] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp, totalSize=230.3 K 2024-11-22T18:56:22,731 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/fc1eb86044524f07bba0c4d38443223f, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/613b1524118e45b8a08f062ba49275b3, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/b59bb2f5872c42e8b929fd704da99738, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/91c76528f20e485982bb5dcf93d5c0f3, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/3d334b3d924c4390a8984aa301c2bdcc, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/95549494c9264735b083aa17ca33bf27, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/95e286c197ee41298f221130f3a58072] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp, totalSize=83.8 K 2024-11-22T18:56:22,731 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=9 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:22,731 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. files: [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/7fe7f94b49884417a715b6e194311664, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/420a9f37d1714f67bb72e2a5c3b88abc, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/4eecacf48a5d45029a48814dbd557b78, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/cb5507c6c6164ebca1f8442c94e6ef50, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/ebc01b2f497b4d12bc7ee03111a98b59, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/f1a786746f2745d881c1d0dac9ccf1b1, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/fc7eb3f7a996421f862800407cee6937] 2024-11-22T18:56:22,731 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting fc1eb86044524f07bba0c4d38443223f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1732301769024 2024-11-22T18:56:22,732 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7fe7f94b49884417a715b6e194311664, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1732301769024 2024-11-22T18:56:22,732 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 613b1524118e45b8a08f062ba49275b3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732301771233 2024-11-22T18:56:22,732 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 420a9f37d1714f67bb72e2a5c3b88abc, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732301771233 2024-11-22T18:56:22,733 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting b59bb2f5872c42e8b929fd704da99738, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732301773404 2024-11-22T18:56:22,733 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4eecacf48a5d45029a48814dbd557b78, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732301773404 2024-11-22T18:56:22,733 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb5507c6c6164ebca1f8442c94e6ef50, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732301774575 2024-11-22T18:56:22,733 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 91c76528f20e485982bb5dcf93d5c0f3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732301774575 2024-11-22T18:56:22,733 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting ebc01b2f497b4d12bc7ee03111a98b59, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732301776734 2024-11-22T18:56:22,733 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d334b3d924c4390a8984aa301c2bdcc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732301776734 2024-11-22T18:56:22,734 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 95549494c9264735b083aa17ca33bf27, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732301778914 2024-11-22T18:56:22,734 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1a786746f2745d881c1d0dac9ccf1b1, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732301778914 2024-11-22T18:56:22,734 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 95e286c197ee41298f221130f3a58072, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732301781088 2024-11-22T18:56:22,735 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc7eb3f7a996421f862800407cee6937, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732301781083 2024-11-22T18:56:22,753 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e2eea2cf5b56baf2f173c11708a11448#B#compaction#162 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:56:22,754 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/B/5bb3be196eb94cabb3cf6040a2c3c6e6 is 50, key is test_row_0/B:col10/1732301782239/Put/seqid=0 2024-11-22T18:56:22,755 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e2eea2cf5b56baf2f173c11708a11448] 2024-11-22T18:56:22,757 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241122cd2d98211f6240f1b043180f6904f61f_e2eea2cf5b56baf2f173c11708a11448 store=[table=TestAcidGuarantees family=A region=e2eea2cf5b56baf2f173c11708a11448] 2024-11-22T18:56:22,760 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241122cd2d98211f6240f1b043180f6904f61f_e2eea2cf5b56baf2f173c11708a11448, store=[table=TestAcidGuarantees family=A region=e2eea2cf5b56baf2f173c11708a11448] 2024-11-22T18:56:22,760 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122cd2d98211f6240f1b043180f6904f61f_e2eea2cf5b56baf2f173c11708a11448 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e2eea2cf5b56baf2f173c11708a11448] 2024-11-22T18:56:22,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742021_1197 (size=12949) 2024-11-22T18:56:22,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742022_1198 (size=4469) 2024-11-22T18:56:22,778 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e2eea2cf5b56baf2f173c11708a11448#A#compaction#163 average throughput is 1.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:56:22,778 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/7cd2d2b566984d3483fa9adfc0fba23e is 175, key is test_row_0/A:col10/1732301782239/Put/seqid=0 2024-11-22T18:56:22,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742023_1199 (size=31903) 2024-11-22T18:56:22,789 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/7cd2d2b566984d3483fa9adfc0fba23e as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/7cd2d2b566984d3483fa9adfc0fba23e 2024-11-22T18:56:22,794 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 7 (all) file(s) in e2eea2cf5b56baf2f173c11708a11448/A of e2eea2cf5b56baf2f173c11708a11448 into 7cd2d2b566984d3483fa9adfc0fba23e(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:56:22,794 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e2eea2cf5b56baf2f173c11708a11448: 2024-11-22T18:56:22,794 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., storeName=e2eea2cf5b56baf2f173c11708a11448/A, priority=9, startTime=1732301782728; duration=0sec 2024-11-22T18:56:22,794 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:56:22,794 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e2eea2cf5b56baf2f173c11708a11448:A 2024-11-22T18:56:22,794 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 0 compacting, 7 eligible, 16 blocking 2024-11-22T18:56:22,796 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 7 files of size 85767 starting at candidate #0 after considering 15 permutations with 15 in ratio 2024-11-22T18:56:22,796 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1541): e2eea2cf5b56baf2f173c11708a11448/C is initiating minor compaction (all files) 2024-11-22T18:56:22,796 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e2eea2cf5b56baf2f173c11708a11448/C in TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:22,796 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/91c8c4cf5aae45a98b3830dd59046350, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/f28de847f7a648a7970a141330250056, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/87f173447ece4f9da5aab197dc9af1a9, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/a4441327641a432187b040741a155605, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/98b88b2c22464feb92d6f610849dfab7, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/2a791ea2f7914ae4b31f31a2287e2580, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/33b729e14c094aa9b3089c48997f20c2] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp, totalSize=83.8 K 2024-11-22T18:56:22,797 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91c8c4cf5aae45a98b3830dd59046350, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1732301769024 2024-11-22T18:56:22,797 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting f28de847f7a648a7970a141330250056, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732301771233 2024-11-22T18:56:22,798 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87f173447ece4f9da5aab197dc9af1a9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732301773404 2024-11-22T18:56:22,799 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4441327641a432187b040741a155605, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732301774575 2024-11-22T18:56:22,800 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98b88b2c22464feb92d6f610849dfab7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732301776734 2024-11-22T18:56:22,800 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a791ea2f7914ae4b31f31a2287e2580, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732301778914 2024-11-22T18:56:22,800 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33b729e14c094aa9b3089c48997f20c2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732301781088 2024-11-22T18:56:22,815 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e2eea2cf5b56baf2f173c11708a11448#C#compaction#164 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:56:22,815 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/C/c7a6f7b79ba2446a9fb5261adfb8c124 is 50, key is test_row_0/C:col10/1732301782239/Put/seqid=0 2024-11-22T18:56:22,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742024_1200 (size=12949) 2024-11-22T18:56:22,828 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/C/c7a6f7b79ba2446a9fb5261adfb8c124 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/c7a6f7b79ba2446a9fb5261adfb8c124 2024-11-22T18:56:22,835 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 7 (all) file(s) in e2eea2cf5b56baf2f173c11708a11448/C of e2eea2cf5b56baf2f173c11708a11448 into c7a6f7b79ba2446a9fb5261adfb8c124(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:56:22,835 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e2eea2cf5b56baf2f173c11708a11448: 2024-11-22T18:56:22,835 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., storeName=e2eea2cf5b56baf2f173c11708a11448/C, priority=9, startTime=1732301782728; duration=0sec 2024-11-22T18:56:22,835 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:56:22,835 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e2eea2cf5b56baf2f173c11708a11448:C 2024-11-22T18:56:22,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:22,887 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T18:56:22,887 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e2eea2cf5b56baf2f173c11708a11448 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-22T18:56:22,894 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112259635722e8cf4b28923a3b89f9705dac_e2eea2cf5b56baf2f173c11708a11448 is 50, key is test_row_0/A:col10/1732301782885/Put/seqid=0 2024-11-22T18:56:22,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742025_1201 (size=14994) 2024-11-22T18:56:22,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-22T18:56:22,905 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T18:56:22,906 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T18:56:22,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-22T18:56:22,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-22T18:56:22,908 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T18:56:22,908 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T18:56:22,909 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T18:56:22,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:22,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43494 deadline: 1732301842912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:22,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:22,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:22,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:22,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:22,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43508 deadline: 1732301842913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:22,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:22,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:22,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:22,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:22,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43426 deadline: 1732301842920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:22,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:22,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43478 deadline: 1732301842922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:22,925 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:22,925 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:22,925 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:22,925 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:22,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:22,925 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:22,925 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:22,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43470 deadline: 1732301842923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:22,925 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:22,925 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:22,925 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:23,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-22T18:56:23,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:23,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43494 deadline: 1732301843025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:23,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:23,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:23,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:23,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:23,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43508 deadline: 1732301843035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:23,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:23,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:23,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:23,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:23,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43426 deadline: 1732301843035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:23,038 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:23,038 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:23,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:23,038 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:23,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43478 deadline: 1732301843036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:23,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:23,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:23,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:23,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:23,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43470 deadline: 1732301843036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:23,040 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:23,040 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:23,040 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:23,050 INFO [Thread-625 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-625. 2024-11-22T18:56:23,050 DEBUG [Thread-625 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T18:56:23,050 DEBUG [Thread-625 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:56:23,051 DEBUG [Thread-625 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:56:23,051 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T18:56:23,053 INFO [Thread-623 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-623. 2024-11-22T18:56:23,054 DEBUG [Thread-623 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T18:56:23,054 DEBUG [Thread-623 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:56:23,054 DEBUG [Thread-623 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:56:23,054 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T18:56:23,056 INFO [Thread-629 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-629. 2024-11-22T18:56:23,056 DEBUG [Thread-629 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T18:56:23,056 DEBUG [Thread-629 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:56:23,056 DEBUG [Thread-629 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:56:23,056 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T18:56:23,060 INFO [Thread-627 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-627. 2024-11-22T18:56:23,060 DEBUG [Thread-627 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T18:56:23,060 DEBUG [Thread-627 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:56:23,061 DEBUG [Thread-627 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:56:23,061 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T18:56:23,061 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-22T18:56:23,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:23,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. as already flushing 2024-11-22T18:56:23,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:23,062 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:23,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:23,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:23,172 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/B/5bb3be196eb94cabb3cf6040a2c3c6e6 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/5bb3be196eb94cabb3cf6040a2c3c6e6 2024-11-22T18:56:23,176 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 7 (all) file(s) in e2eea2cf5b56baf2f173c11708a11448/B of e2eea2cf5b56baf2f173c11708a11448 into 5bb3be196eb94cabb3cf6040a2c3c6e6(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:56:23,176 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e2eea2cf5b56baf2f173c11708a11448: 2024-11-22T18:56:23,176 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., storeName=e2eea2cf5b56baf2f173c11708a11448/B, priority=9, startTime=1732301782728; duration=0sec 2024-11-22T18:56:23,176 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:56:23,176 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e2eea2cf5b56baf2f173c11708a11448:B 2024-11-22T18:56:23,215 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-22T18:56:23,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:23,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. as already flushing 2024-11-22T18:56:23,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:23,215 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:23,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:23,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:23,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-22T18:56:23,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:23,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43494 deadline: 1732301843235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:23,236 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:23,236 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:23,236 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:23,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:23,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:23,246 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:23,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43508 deadline: 1732301843245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:23,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43426 deadline: 1732301843245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:23,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43478 deadline: 1732301843246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:23,246 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:23,246 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:23,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:23,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:23,246 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:23,246 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:23,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:23,246 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:23,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:23,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43470 deadline: 1732301843246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:23,246 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:23,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:23,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:23,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:23,300 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:56:23,304 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112259635722e8cf4b28923a3b89f9705dac_e2eea2cf5b56baf2f173c11708a11448 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112259635722e8cf4b28923a3b89f9705dac_e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:23,305 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/f682ae7a801941049ca348cb8394dca7, store: [table=TestAcidGuarantees family=A region=e2eea2cf5b56baf2f173c11708a11448] 2024-11-22T18:56:23,305 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/f682ae7a801941049ca348cb8394dca7 is 175, key is test_row_0/A:col10/1732301782885/Put/seqid=0 2024-11-22T18:56:23,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742026_1202 (size=39949) 2024-11-22T18:56:23,368 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-22T18:56:23,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:23,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. as already flushing 2024-11-22T18:56:23,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:23,368 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:23,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:23,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:23,521 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-22T18:56:23,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:23,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. as already flushing 2024-11-22T18:56:23,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:23,522 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:23,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:23,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:23,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-22T18:56:23,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:23,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43494 deadline: 1732301843546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:23,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:23,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:23,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:23,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:23,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43426 deadline: 1732301843555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:23,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:23,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:23,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43508 deadline: 1732301843555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:23,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43478 deadline: 1732301843555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:23,556 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:23,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:23,556 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:23,556 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:23,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43470 deadline: 1732301843555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:23,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:23,556 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:23,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:23,556 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:23,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:23,556 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:23,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:23,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:23,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:23,674 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-22T18:56:23,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:23,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. as already flushing 2024-11-22T18:56:23,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:23,674 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:23,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:23,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:23,710 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=310, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/f682ae7a801941049ca348cb8394dca7 2024-11-22T18:56:23,717 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/B/b29afb772bba44f794ade8e5db9149b9 is 50, key is test_row_0/B:col10/1732301782885/Put/seqid=0 2024-11-22T18:56:23,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742027_1203 (size=12301) 2024-11-22T18:56:23,827 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-22T18:56:23,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:23,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. as already flushing 2024-11-22T18:56:23,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:23,828 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:23,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:23,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:23,979 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-22T18:56:23,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:23,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. as already flushing 2024-11-22T18:56:23,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:23,980 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:23,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:23,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:24,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-22T18:56:24,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:24,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43494 deadline: 1732301844056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:24,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:24,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:24,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:24,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:24,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43426 deadline: 1732301844065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:24,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:24,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:24,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43508 deadline: 1732301844065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:24,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43478 deadline: 1732301844065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:24,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:24,066 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:24,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43470 deadline: 1732301844065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:24,066 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:24,066 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:24,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:24,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:24,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:24,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:24,066 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:24,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:24,066 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:24,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:24,066 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:56:24,121 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/B/b29afb772bba44f794ade8e5db9149b9 2024-11-22T18:56:24,128 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/C/afeb3735731c476cac9ad3506281fdc0 is 50, key is test_row_0/C:col10/1732301782885/Put/seqid=0 2024-11-22T18:56:24,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742028_1204 (size=12301) 2024-11-22T18:56:24,132 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-22T18:56:24,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:24,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. as already flushing 2024-11-22T18:56:24,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:24,133 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:24,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:24,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:24,285 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-22T18:56:24,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:24,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. as already flushing 2024-11-22T18:56:24,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:24,286 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:24,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:24,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:24,439 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-22T18:56:24,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:24,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. as already flushing 2024-11-22T18:56:24,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:24,439 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:24,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:24,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:24,532 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/C/afeb3735731c476cac9ad3506281fdc0 2024-11-22T18:56:24,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/f682ae7a801941049ca348cb8394dca7 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/f682ae7a801941049ca348cb8394dca7 2024-11-22T18:56:24,540 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/f682ae7a801941049ca348cb8394dca7, entries=200, sequenceid=310, filesize=39.0 K 2024-11-22T18:56:24,541 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/B/b29afb772bba44f794ade8e5db9149b9 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/b29afb772bba44f794ade8e5db9149b9 2024-11-22T18:56:24,545 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/b29afb772bba44f794ade8e5db9149b9, entries=150, sequenceid=310, filesize=12.0 K 2024-11-22T18:56:24,545 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/C/afeb3735731c476cac9ad3506281fdc0 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/afeb3735731c476cac9ad3506281fdc0 2024-11-22T18:56:24,549 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/afeb3735731c476cac9ad3506281fdc0, entries=150, sequenceid=310, filesize=12.0 K 2024-11-22T18:56:24,550 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for e2eea2cf5b56baf2f173c11708a11448 in 1663ms, sequenceid=310, compaction requested=false 2024-11-22T18:56:24,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e2eea2cf5b56baf2f173c11708a11448: 2024-11-22T18:56:24,592 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-22T18:56:24,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:24,592 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing e2eea2cf5b56baf2f173c11708a11448 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-22T18:56:24,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411227b32fcb25ac543c79bcad5364bb245b4_e2eea2cf5b56baf2f173c11708a11448 is 50, key is test_row_0/A:col10/1732301782911/Put/seqid=0 2024-11-22T18:56:24,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742029_1205 (size=12454) 2024-11-22T18:56:25,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:56:25,006 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411227b32fcb25ac543c79bcad5364bb245b4_e2eea2cf5b56baf2f173c11708a11448 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411227b32fcb25ac543c79bcad5364bb245b4_e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:25,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/4900564b58874cb899785e70d835d34b, store: [table=TestAcidGuarantees family=A region=e2eea2cf5b56baf2f173c11708a11448] 2024-11-22T18:56:25,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/4900564b58874cb899785e70d835d34b is 175, key is test_row_0/A:col10/1732301782911/Put/seqid=0 2024-11-22T18:56:25,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742030_1206 (size=31255) 2024-11-22T18:56:25,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-22T18:56:25,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:25,067 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. as already flushing 2024-11-22T18:56:25,067 INFO [Thread-612 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-612. 2024-11-22T18:56:25,067 DEBUG [Thread-612 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T18:56:25,067 DEBUG [Thread-612 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:56:25,067 DEBUG [Thread-612 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:56:25,068 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T18:56:25,077 INFO [Thread-616 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-616. 2024-11-22T18:56:25,077 INFO [Thread-618 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-618. 2024-11-22T18:56:25,077 INFO [Thread-620 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-620. 2024-11-22T18:56:25,077 DEBUG [Thread-616 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T18:56:25,077 DEBUG [Thread-616 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:56:25,077 DEBUG [Thread-620 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T18:56:25,077 DEBUG [Thread-618 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T18:56:25,077 DEBUG [Thread-620 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:56:25,077 DEBUG [Thread-618 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:56:25,077 DEBUG [Thread-616 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:56:25,077 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T18:56:25,077 DEBUG [Thread-620 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:56:25,077 DEBUG [Thread-618 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:56:25,077 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T18:56:25,077 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T18:56:25,078 INFO [Thread-614 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-614. 2024-11-22T18:56:25,078 DEBUG [Thread-614 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T18:56:25,078 DEBUG [Thread-614 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:56:25,078 DEBUG [Thread-614 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:56:25,078 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T18:56:25,411 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=330, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/4900564b58874cb899785e70d835d34b 2024-11-22T18:56:25,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/B/2429741929a8479197be75c6aebe2efc is 50, key is test_row_0/B:col10/1732301782911/Put/seqid=0 2024-11-22T18:56:25,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742031_1207 (size=12301) 2024-11-22T18:56:25,822 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/B/2429741929a8479197be75c6aebe2efc 2024-11-22T18:56:25,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/C/4d30a465c70a4f678c4bd32f81194d57 is 50, key is test_row_0/C:col10/1732301782911/Put/seqid=0 2024-11-22T18:56:25,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742032_1208 (size=12301) 2024-11-22T18:56:26,234 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/C/4d30a465c70a4f678c4bd32f81194d57 2024-11-22T18:56:26,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/4900564b58874cb899785e70d835d34b as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/4900564b58874cb899785e70d835d34b 2024-11-22T18:56:26,243 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/4900564b58874cb899785e70d835d34b, entries=150, sequenceid=330, filesize=30.5 K 2024-11-22T18:56:26,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/B/2429741929a8479197be75c6aebe2efc as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/2429741929a8479197be75c6aebe2efc 2024-11-22T18:56:26,247 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/2429741929a8479197be75c6aebe2efc, entries=150, sequenceid=330, filesize=12.0 K 2024-11-22T18:56:26,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/C/4d30a465c70a4f678c4bd32f81194d57 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/4d30a465c70a4f678c4bd32f81194d57 2024-11-22T18:56:26,251 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/4d30a465c70a4f678c4bd32f81194d57, entries=150, sequenceid=330, filesize=12.0 K 2024-11-22T18:56:26,252 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=33.54 KB/34350 for e2eea2cf5b56baf2f173c11708a11448 in 1660ms, sequenceid=330, compaction requested=true 2024-11-22T18:56:26,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for e2eea2cf5b56baf2f173c11708a11448: 2024-11-22T18:56:26,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:26,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-22T18:56:26,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-22T18:56:26,255 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-22T18:56:26,255 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3460 sec 2024-11-22T18:56:26,256 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 3.3490 sec 2024-11-22T18:56:27,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-22T18:56:27,086 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T18:56:27,086 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-22T18:56:27,086 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-11-22T18:56:27,086 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 51 2024-11-22T18:56:27,086 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-11-22T18:56:27,086 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 46 2024-11-22T18:56:27,086 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 62 2024-11-22T18:56:27,086 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-22T18:56:27,086 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3319 2024-11-22T18:56:27,086 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3420 2024-11-22T18:56:27,086 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-22T18:56:27,086 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1450 2024-11-22T18:56:27,086 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4347 rows 2024-11-22T18:56:27,086 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1462 2024-11-22T18:56:27,086 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4379 rows 2024-11-22T18:56:27,086 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-22T18:56:27,086 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T18:56:27,086 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:56:27,086 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:56:27,086 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T18:56:27,087 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-22T18:56:27,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-22T18:56:27,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-22T18:56:27,090 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732301787090"}]},"ts":"1732301787090"} 2024-11-22T18:56:27,091 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-22T18:56:27,091 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-22T18:56:27,092 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-22T18:56:27,093 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e2eea2cf5b56baf2f173c11708a11448, UNASSIGN}] 2024-11-22T18:56:27,094 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e2eea2cf5b56baf2f173c11708a11448, UNASSIGN 2024-11-22T18:56:27,094 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=e2eea2cf5b56baf2f173c11708a11448, regionState=CLOSING, regionLocation=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:27,095 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e2eea2cf5b56baf2f173c11708a11448, UNASSIGN because future has completed 2024-11-22T18:56:27,096 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-22T18:56:27,096 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; CloseRegionProcedure e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173}] 2024-11-22T18:56:27,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-22T18:56:27,248 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=67}] handler.UnassignRegionHandler(122): Close e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:27,248 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=67}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-22T18:56:27,248 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=67}] regionserver.HRegion(1722): Closing e2eea2cf5b56baf2f173c11708a11448, disabling compactions & flushes 2024-11-22T18:56:27,248 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=67}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:27,248 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=67}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:27,248 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=67}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. after waiting 0 ms 2024-11-22T18:56:27,248 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=67}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:27,248 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=67}] regionserver.HRegion(2902): Flushing e2eea2cf5b56baf2f173c11708a11448 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-22T18:56:27,254 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112282616de32a6a4a50b5c42a88792fdc13_e2eea2cf5b56baf2f173c11708a11448 is 50, key is test_row_0/A:col10/1732301785075/Put/seqid=0 2024-11-22T18:56:27,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742033_1209 (size=9914) 2024-11-22T18:56:27,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-22T18:56:27,658 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=67}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:56:27,662 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=67}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112282616de32a6a4a50b5c42a88792fdc13_e2eea2cf5b56baf2f173c11708a11448 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112282616de32a6a4a50b5c42a88792fdc13_e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:27,663 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=67}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/d99ad3a9aadd466cae764193f96519d8, store: [table=TestAcidGuarantees family=A region=e2eea2cf5b56baf2f173c11708a11448] 2024-11-22T18:56:27,663 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/d99ad3a9aadd466cae764193f96519d8 is 175, key is test_row_0/A:col10/1732301785075/Put/seqid=0 2024-11-22T18:56:27,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742034_1210 (size=22561) 2024-11-22T18:56:27,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-22T18:56:28,068 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=67}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=338, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/d99ad3a9aadd466cae764193f96519d8 2024-11-22T18:56:28,075 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/B/9b0f9b291f0041fe8e20c5d541c2572e is 50, key is test_row_0/B:col10/1732301785075/Put/seqid=0 2024-11-22T18:56:28,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742035_1211 (size=9857) 2024-11-22T18:56:28,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-22T18:56:28,479 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/B/9b0f9b291f0041fe8e20c5d541c2572e 2024-11-22T18:56:28,486 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/C/b018f601def54cabb59fd879149d6b5c is 50, key is test_row_0/C:col10/1732301785075/Put/seqid=0 2024-11-22T18:56:28,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742036_1212 (size=9857) 2024-11-22T18:56:28,890 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/C/b018f601def54cabb59fd879149d6b5c 2024-11-22T18:56:28,895 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/A/d99ad3a9aadd466cae764193f96519d8 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/d99ad3a9aadd466cae764193f96519d8 2024-11-22T18:56:28,899 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/d99ad3a9aadd466cae764193f96519d8, entries=100, sequenceid=338, filesize=22.0 K 2024-11-22T18:56:28,899 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/B/9b0f9b291f0041fe8e20c5d541c2572e as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/9b0f9b291f0041fe8e20c5d541c2572e 2024-11-22T18:56:28,903 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/9b0f9b291f0041fe8e20c5d541c2572e, entries=100, sequenceid=338, filesize=9.6 K 2024-11-22T18:56:28,903 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/.tmp/C/b018f601def54cabb59fd879149d6b5c as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/b018f601def54cabb59fd879149d6b5c 2024-11-22T18:56:28,906 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/b018f601def54cabb59fd879149d6b5c, entries=100, sequenceid=338, filesize=9.6 K 2024-11-22T18:56:28,907 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for e2eea2cf5b56baf2f173c11708a11448 in 1659ms, sequenceid=338, compaction requested=true 2024-11-22T18:56:28,908 DEBUG [StoreCloser-TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/403ce19885a84436976f47c5f56abe64, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/28d67d7ebd594de5bb52acf3ee65ba3d, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/1536a8e4f6244044a3a8775d1433cd72, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/940fa24840b040d594a798075f846816, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/3debbb0a5d654b379d049f0a4e66664b, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/fdffb78f50654d9d906b180742e0d74c, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/064a04273c704727917d3c19e1f6c55c, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/a67d1b35cae8491e864393b25dd6cf59, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/4a4b68be641a4316a96dad951bab0a1d, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/cfe89cd0dd704203a578114fbd0b43f7, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/2852d5900da449429f6be52be2dc9a29, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/dc59fb3313594649abd5f526d3f8dd93, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/7fe7f94b49884417a715b6e194311664, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/420a9f37d1714f67bb72e2a5c3b88abc, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/4eecacf48a5d45029a48814dbd557b78, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/cb5507c6c6164ebca1f8442c94e6ef50, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/ebc01b2f497b4d12bc7ee03111a98b59, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/f1a786746f2745d881c1d0dac9ccf1b1, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/fc7eb3f7a996421f862800407cee6937] to archive 2024-11-22T18:56:28,908 DEBUG [StoreCloser-TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T18:56:28,910 DEBUG [StoreCloser-TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/403ce19885a84436976f47c5f56abe64 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/403ce19885a84436976f47c5f56abe64 2024-11-22T18:56:28,911 DEBUG [StoreCloser-TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/28d67d7ebd594de5bb52acf3ee65ba3d to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/28d67d7ebd594de5bb52acf3ee65ba3d 2024-11-22T18:56:28,912 DEBUG [StoreCloser-TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/1536a8e4f6244044a3a8775d1433cd72 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/1536a8e4f6244044a3a8775d1433cd72 2024-11-22T18:56:28,913 DEBUG [StoreCloser-TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/940fa24840b040d594a798075f846816 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/940fa24840b040d594a798075f846816 2024-11-22T18:56:28,914 DEBUG [StoreCloser-TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/3debbb0a5d654b379d049f0a4e66664b to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/3debbb0a5d654b379d049f0a4e66664b 2024-11-22T18:56:28,915 DEBUG [StoreCloser-TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/fdffb78f50654d9d906b180742e0d74c to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/fdffb78f50654d9d906b180742e0d74c 2024-11-22T18:56:28,916 DEBUG [StoreCloser-TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/064a04273c704727917d3c19e1f6c55c to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/064a04273c704727917d3c19e1f6c55c 2024-11-22T18:56:28,917 DEBUG [StoreCloser-TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/a67d1b35cae8491e864393b25dd6cf59 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/a67d1b35cae8491e864393b25dd6cf59 2024-11-22T18:56:28,918 DEBUG [StoreCloser-TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/4a4b68be641a4316a96dad951bab0a1d to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/4a4b68be641a4316a96dad951bab0a1d 2024-11-22T18:56:28,919 DEBUG [StoreCloser-TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/cfe89cd0dd704203a578114fbd0b43f7 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/cfe89cd0dd704203a578114fbd0b43f7 2024-11-22T18:56:28,920 DEBUG [StoreCloser-TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/2852d5900da449429f6be52be2dc9a29 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/2852d5900da449429f6be52be2dc9a29 2024-11-22T18:56:28,921 DEBUG [StoreCloser-TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/dc59fb3313594649abd5f526d3f8dd93 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/dc59fb3313594649abd5f526d3f8dd93 2024-11-22T18:56:28,922 DEBUG [StoreCloser-TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/7fe7f94b49884417a715b6e194311664 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/7fe7f94b49884417a715b6e194311664 2024-11-22T18:56:28,923 DEBUG [StoreCloser-TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/420a9f37d1714f67bb72e2a5c3b88abc to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/420a9f37d1714f67bb72e2a5c3b88abc 2024-11-22T18:56:28,924 DEBUG [StoreCloser-TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/4eecacf48a5d45029a48814dbd557b78 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/4eecacf48a5d45029a48814dbd557b78 2024-11-22T18:56:28,925 DEBUG [StoreCloser-TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/cb5507c6c6164ebca1f8442c94e6ef50 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/cb5507c6c6164ebca1f8442c94e6ef50 2024-11-22T18:56:28,926 DEBUG [StoreCloser-TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/ebc01b2f497b4d12bc7ee03111a98b59 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/ebc01b2f497b4d12bc7ee03111a98b59 2024-11-22T18:56:28,927 DEBUG [StoreCloser-TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/f1a786746f2745d881c1d0dac9ccf1b1 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/f1a786746f2745d881c1d0dac9ccf1b1 2024-11-22T18:56:28,928 DEBUG [StoreCloser-TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/fc7eb3f7a996421f862800407cee6937 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/fc7eb3f7a996421f862800407cee6937 2024-11-22T18:56:28,929 DEBUG [StoreCloser-TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/138af5913f8847bf9addf3a422eab171, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/4e80a32fabf045839b400a806e9d736c, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/c2d87713cc8a4848b9f9aa047d033938, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/ec421512489d41989ba0939732a9b363, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/e621a93964d14ed5a6603552411c70a3, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/c7b303c54b8c44889eb06ec4cff6c487, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/4bf4f02a9d39482fbdae70c98d19ad8f, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/daba81a966e94ddab26f7e1ce57bfd38, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/008449021d554a3d8ec2c6790c8c2a05, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/b12db7b782a64d8b8e7897248cd59d0a, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/fa9f7ad7fe79436183200ca58e6d185f, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/fc1eb86044524f07bba0c4d38443223f, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/d97eb7ee240b422bb348b651d40c93db, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/613b1524118e45b8a08f062ba49275b3, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/b59bb2f5872c42e8b929fd704da99738, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/91c76528f20e485982bb5dcf93d5c0f3, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/3d334b3d924c4390a8984aa301c2bdcc, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/95549494c9264735b083aa17ca33bf27, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/95e286c197ee41298f221130f3a58072] to archive 2024-11-22T18:56:28,930 DEBUG [StoreCloser-TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T18:56:28,931 DEBUG [StoreCloser-TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/138af5913f8847bf9addf3a422eab171 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/138af5913f8847bf9addf3a422eab171 2024-11-22T18:56:28,932 DEBUG [StoreCloser-TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/4e80a32fabf045839b400a806e9d736c to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/4e80a32fabf045839b400a806e9d736c 2024-11-22T18:56:28,933 DEBUG [StoreCloser-TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/c2d87713cc8a4848b9f9aa047d033938 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/c2d87713cc8a4848b9f9aa047d033938 2024-11-22T18:56:28,934 DEBUG [StoreCloser-TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/ec421512489d41989ba0939732a9b363 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/ec421512489d41989ba0939732a9b363 2024-11-22T18:56:28,935 DEBUG [StoreCloser-TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/e621a93964d14ed5a6603552411c70a3 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/e621a93964d14ed5a6603552411c70a3 2024-11-22T18:56:28,936 DEBUG [StoreCloser-TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/c7b303c54b8c44889eb06ec4cff6c487 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/c7b303c54b8c44889eb06ec4cff6c487 2024-11-22T18:56:28,937 DEBUG [StoreCloser-TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/4bf4f02a9d39482fbdae70c98d19ad8f to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/4bf4f02a9d39482fbdae70c98d19ad8f 2024-11-22T18:56:28,938 DEBUG [StoreCloser-TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/daba81a966e94ddab26f7e1ce57bfd38 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/daba81a966e94ddab26f7e1ce57bfd38 2024-11-22T18:56:28,939 DEBUG [StoreCloser-TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/008449021d554a3d8ec2c6790c8c2a05 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/008449021d554a3d8ec2c6790c8c2a05 2024-11-22T18:56:28,940 DEBUG [StoreCloser-TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/b12db7b782a64d8b8e7897248cd59d0a to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/b12db7b782a64d8b8e7897248cd59d0a 2024-11-22T18:56:28,941 DEBUG [StoreCloser-TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/fa9f7ad7fe79436183200ca58e6d185f to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/fa9f7ad7fe79436183200ca58e6d185f 2024-11-22T18:56:28,942 DEBUG [StoreCloser-TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/fc1eb86044524f07bba0c4d38443223f to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/fc1eb86044524f07bba0c4d38443223f 2024-11-22T18:56:28,943 DEBUG [StoreCloser-TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/d97eb7ee240b422bb348b651d40c93db to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/d97eb7ee240b422bb348b651d40c93db 2024-11-22T18:56:28,944 DEBUG [StoreCloser-TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/613b1524118e45b8a08f062ba49275b3 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/613b1524118e45b8a08f062ba49275b3 2024-11-22T18:56:28,945 DEBUG [StoreCloser-TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/b59bb2f5872c42e8b929fd704da99738 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/b59bb2f5872c42e8b929fd704da99738 2024-11-22T18:56:28,946 DEBUG [StoreCloser-TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/91c76528f20e485982bb5dcf93d5c0f3 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/91c76528f20e485982bb5dcf93d5c0f3 2024-11-22T18:56:28,947 DEBUG [StoreCloser-TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/3d334b3d924c4390a8984aa301c2bdcc to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/3d334b3d924c4390a8984aa301c2bdcc 2024-11-22T18:56:28,947 DEBUG [StoreCloser-TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/95549494c9264735b083aa17ca33bf27 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/95549494c9264735b083aa17ca33bf27 2024-11-22T18:56:28,948 DEBUG [StoreCloser-TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/95e286c197ee41298f221130f3a58072 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/95e286c197ee41298f221130f3a58072 2024-11-22T18:56:28,950 DEBUG [StoreCloser-TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/41d539f400c44763956b04a79a5acd13, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/8918033e41ea4dab845f8675e3613969, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/06d7d604bcbf4e11ab8dfe11bd7816bf, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/846a945de0814246b5321b24117dd757, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/209b6ce9d2434e268abbcebe479b388b, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/16e4272972eb4ead8fda36217b16b356, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/3362a1202bfa4edaae5daf7378c8ce98, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/a6d069f36f9641eabee962b77fa8ad4b, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/97a0cbdee4ae4767a912c3f26e9ea79e, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/2e4447bd01ba46b69667d63409b5848a, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/60ccdfec1a594d74866ffbfbfc98f50a, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/91c8c4cf5aae45a98b3830dd59046350, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/76c09e1e95c143889744717feb96d876, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/f28de847f7a648a7970a141330250056, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/87f173447ece4f9da5aab197dc9af1a9, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/a4441327641a432187b040741a155605, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/98b88b2c22464feb92d6f610849dfab7, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/2a791ea2f7914ae4b31f31a2287e2580, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/33b729e14c094aa9b3089c48997f20c2] to archive 2024-11-22T18:56:28,951 DEBUG [StoreCloser-TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T18:56:28,952 DEBUG [StoreCloser-TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/41d539f400c44763956b04a79a5acd13 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/41d539f400c44763956b04a79a5acd13 2024-11-22T18:56:28,953 DEBUG [StoreCloser-TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/8918033e41ea4dab845f8675e3613969 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/8918033e41ea4dab845f8675e3613969 2024-11-22T18:56:28,954 DEBUG [StoreCloser-TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/06d7d604bcbf4e11ab8dfe11bd7816bf to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/06d7d604bcbf4e11ab8dfe11bd7816bf 2024-11-22T18:56:28,955 DEBUG [StoreCloser-TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/846a945de0814246b5321b24117dd757 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/846a945de0814246b5321b24117dd757 2024-11-22T18:56:28,956 DEBUG [StoreCloser-TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/209b6ce9d2434e268abbcebe479b388b to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/209b6ce9d2434e268abbcebe479b388b 2024-11-22T18:56:28,957 DEBUG [StoreCloser-TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/16e4272972eb4ead8fda36217b16b356 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/16e4272972eb4ead8fda36217b16b356 2024-11-22T18:56:28,957 DEBUG [StoreCloser-TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/3362a1202bfa4edaae5daf7378c8ce98 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/3362a1202bfa4edaae5daf7378c8ce98 2024-11-22T18:56:28,958 DEBUG [StoreCloser-TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/a6d069f36f9641eabee962b77fa8ad4b to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/a6d069f36f9641eabee962b77fa8ad4b 2024-11-22T18:56:28,959 DEBUG [StoreCloser-TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/97a0cbdee4ae4767a912c3f26e9ea79e to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/97a0cbdee4ae4767a912c3f26e9ea79e 2024-11-22T18:56:28,960 DEBUG [StoreCloser-TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/2e4447bd01ba46b69667d63409b5848a to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/2e4447bd01ba46b69667d63409b5848a 2024-11-22T18:56:28,961 DEBUG [StoreCloser-TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/60ccdfec1a594d74866ffbfbfc98f50a to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/60ccdfec1a594d74866ffbfbfc98f50a 2024-11-22T18:56:28,962 DEBUG [StoreCloser-TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/91c8c4cf5aae45a98b3830dd59046350 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/91c8c4cf5aae45a98b3830dd59046350 2024-11-22T18:56:28,962 DEBUG [StoreCloser-TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/76c09e1e95c143889744717feb96d876 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/76c09e1e95c143889744717feb96d876 2024-11-22T18:56:28,963 DEBUG [StoreCloser-TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/f28de847f7a648a7970a141330250056 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/f28de847f7a648a7970a141330250056 2024-11-22T18:56:28,964 DEBUG [StoreCloser-TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/87f173447ece4f9da5aab197dc9af1a9 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/87f173447ece4f9da5aab197dc9af1a9 2024-11-22T18:56:28,965 DEBUG [StoreCloser-TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/a4441327641a432187b040741a155605 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/a4441327641a432187b040741a155605 2024-11-22T18:56:28,966 DEBUG [StoreCloser-TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/98b88b2c22464feb92d6f610849dfab7 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/98b88b2c22464feb92d6f610849dfab7 2024-11-22T18:56:28,967 DEBUG [StoreCloser-TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/2a791ea2f7914ae4b31f31a2287e2580 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/2a791ea2f7914ae4b31f31a2287e2580 2024-11-22T18:56:28,967 DEBUG [StoreCloser-TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/33b729e14c094aa9b3089c48997f20c2 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/33b729e14c094aa9b3089c48997f20c2 2024-11-22T18:56:28,971 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=67}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/recovered.edits/341.seqid, newMaxSeqId=341, maxSeqId=4 2024-11-22T18:56:28,972 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=67}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. 2024-11-22T18:56:28,972 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=67}] regionserver.HRegion(1676): Region close journal for e2eea2cf5b56baf2f173c11708a11448: Waiting for close lock at 1732301787248Running coprocessor pre-close hooks at 1732301787248Disabling compacts and flushes for region at 1732301787248Disabling writes for close at 1732301787248Obtaining lock to block concurrent updates at 1732301787248Preparing flush snapshotting stores in e2eea2cf5b56baf2f173c11708a11448 at 1732301787248Finished memstore snapshotting TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1732301787248Flushing stores of TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448. at 1732301787249 (+1 ms)Flushing e2eea2cf5b56baf2f173c11708a11448/A: creating writer at 1732301787249Flushing mob file e2eea2cf5b56baf2f173c11708a11448/A: appending metadata at 1732301787254 (+5 ms)Flushing mob file e2eea2cf5b56baf2f173c11708a11448/A: closing flushed file at 1732301787254Flushing e2eea2cf5b56baf2f173c11708a11448/A: appending metadata at 1732301787663 (+409 ms)Flushing e2eea2cf5b56baf2f173c11708a11448/A: closing flushed file at 1732301787663Flushing e2eea2cf5b56baf2f173c11708a11448/B: creating writer at 1732301788072 (+409 ms)Flushing e2eea2cf5b56baf2f173c11708a11448/B: appending metadata at 1732301788074 (+2 ms)Flushing e2eea2cf5b56baf2f173c11708a11448/B: closing flushed file at 1732301788074Flushing e2eea2cf5b56baf2f173c11708a11448/C: creating writer at 1732301788483 (+409 ms)Flushing e2eea2cf5b56baf2f173c11708a11448/C: appending metadata at 1732301788485 (+2 ms)Flushing e2eea2cf5b56baf2f173c11708a11448/C: closing flushed file at 1732301788486 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@74a6f2cb: reopening flushed file at 1732301788894 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@9e7fd51: reopening flushed file at 1732301788899 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@92c6830: reopening flushed file at 1732301788903 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for e2eea2cf5b56baf2f173c11708a11448 in 1659ms, sequenceid=338, compaction requested=true at 1732301788907 (+4 ms)Writing region close event to WAL at 1732301788969 (+62 ms)Running coprocessor post-close hooks at 1732301788972 (+3 ms)Closed at 1732301788972 2024-11-22T18:56:28,973 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=67}] handler.UnassignRegionHandler(157): Closed e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:28,974 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=e2eea2cf5b56baf2f173c11708a11448, regionState=CLOSED 2024-11-22T18:56:28,975 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=66, state=RUNNABLE, hasLock=false; CloseRegionProcedure e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 because future has completed 2024-11-22T18:56:28,977 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-22T18:56:28,977 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; CloseRegionProcedure e2eea2cf5b56baf2f173c11708a11448, server=d79ba0c344fb,43593,1732301724173 in 1.8800 sec 2024-11-22T18:56:28,978 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=66, resume processing ppid=65 2024-11-22T18:56:28,978 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, ppid=65, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e2eea2cf5b56baf2f173c11708a11448, UNASSIGN in 1.8840 sec 2024-11-22T18:56:28,980 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-22T18:56:28,980 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8870 sec 2024-11-22T18:56:28,981 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732301788981"}]},"ts":"1732301788981"} 2024-11-22T18:56:28,982 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-22T18:56:28,982 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-22T18:56:28,984 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8960 sec 2024-11-22T18:56:29,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-22T18:56:29,236 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-22T18:56:29,236 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-22T18:56:29,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T18:56:29,238 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=68, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T18:56:29,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-22T18:56:29,239 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=68, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T18:56:29,241 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:29,243 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A, FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B, FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C, FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/recovered.edits] 2024-11-22T18:56:29,246 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/4900564b58874cb899785e70d835d34b to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/4900564b58874cb899785e70d835d34b 2024-11-22T18:56:29,247 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/7cd2d2b566984d3483fa9adfc0fba23e to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/7cd2d2b566984d3483fa9adfc0fba23e 2024-11-22T18:56:29,248 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/d99ad3a9aadd466cae764193f96519d8 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/d99ad3a9aadd466cae764193f96519d8 2024-11-22T18:56:29,249 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/f682ae7a801941049ca348cb8394dca7 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/A/f682ae7a801941049ca348cb8394dca7 2024-11-22T18:56:29,251 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/2429741929a8479197be75c6aebe2efc to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/2429741929a8479197be75c6aebe2efc 2024-11-22T18:56:29,252 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/5bb3be196eb94cabb3cf6040a2c3c6e6 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/5bb3be196eb94cabb3cf6040a2c3c6e6 2024-11-22T18:56:29,253 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/9b0f9b291f0041fe8e20c5d541c2572e to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/9b0f9b291f0041fe8e20c5d541c2572e 2024-11-22T18:56:29,254 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/b29afb772bba44f794ade8e5db9149b9 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/B/b29afb772bba44f794ade8e5db9149b9 2024-11-22T18:56:29,257 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/4d30a465c70a4f678c4bd32f81194d57 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/4d30a465c70a4f678c4bd32f81194d57 2024-11-22T18:56:29,258 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/afeb3735731c476cac9ad3506281fdc0 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/afeb3735731c476cac9ad3506281fdc0 2024-11-22T18:56:29,259 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/b018f601def54cabb59fd879149d6b5c to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/b018f601def54cabb59fd879149d6b5c 2024-11-22T18:56:29,260 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/c7a6f7b79ba2446a9fb5261adfb8c124 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/C/c7a6f7b79ba2446a9fb5261adfb8c124 2024-11-22T18:56:29,262 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/recovered.edits/341.seqid to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448/recovered.edits/341.seqid 2024-11-22T18:56:29,262 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:29,263 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-22T18:56:29,263 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-22T18:56:29,264 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-22T18:56:29,267 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112208e568be8bee427cbdceb1f8e6287346_e2eea2cf5b56baf2f173c11708a11448 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112208e568be8bee427cbdceb1f8e6287346_e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:29,268 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411220d0dc1af500b4eeda891bb17c1d12e76_e2eea2cf5b56baf2f173c11708a11448 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411220d0dc1af500b4eeda891bb17c1d12e76_e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:29,269 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411222406b56a48c244abbc5ac1bff479fa36_e2eea2cf5b56baf2f173c11708a11448 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411222406b56a48c244abbc5ac1bff479fa36_e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:29,270 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411222d9addab35244f24b434ff3a858fb96a_e2eea2cf5b56baf2f173c11708a11448 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411222d9addab35244f24b434ff3a858fb96a_e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:29,272 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112234f70a8bdae94c2da4859a135e5fa5e1_e2eea2cf5b56baf2f173c11708a11448 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112234f70a8bdae94c2da4859a135e5fa5e1_e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:29,273 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411223c0e205bd49248679dbea3a2354be0ca_e2eea2cf5b56baf2f173c11708a11448 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411223c0e205bd49248679dbea3a2354be0ca_e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:29,274 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112259635722e8cf4b28923a3b89f9705dac_e2eea2cf5b56baf2f173c11708a11448 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112259635722e8cf4b28923a3b89f9705dac_e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:29,275 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411225c9ba74b11e04114a769072ec0419bed_e2eea2cf5b56baf2f173c11708a11448 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411225c9ba74b11e04114a769072ec0419bed_e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:29,276 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112276b35f65a2b046e2aa2792dad32c5c50_e2eea2cf5b56baf2f173c11708a11448 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112276b35f65a2b046e2aa2792dad32c5c50_e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:29,277 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411227b32fcb25ac543c79bcad5364bb245b4_e2eea2cf5b56baf2f173c11708a11448 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411227b32fcb25ac543c79bcad5364bb245b4_e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:29,278 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112282616de32a6a4a50b5c42a88792fdc13_e2eea2cf5b56baf2f173c11708a11448 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112282616de32a6a4a50b5c42a88792fdc13_e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:29,279 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411228bd39e0c6416411e9e5d594f3624e836_e2eea2cf5b56baf2f173c11708a11448 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411228bd39e0c6416411e9e5d594f3624e836_e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:29,280 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122a03f438e4640474695b5f017be3728b6_e2eea2cf5b56baf2f173c11708a11448 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122a03f438e4640474695b5f017be3728b6_e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:29,281 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122a10eafd0c8664593a2dd7b544c7f6df7_e2eea2cf5b56baf2f173c11708a11448 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122a10eafd0c8664593a2dd7b544c7f6df7_e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:29,283 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122ba28e75f72784bbf8e45f6dc21980c64_e2eea2cf5b56baf2f173c11708a11448 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122ba28e75f72784bbf8e45f6dc21980c64_e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:29,284 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122c489caf40f1b42a5aff4675f81ae9230_e2eea2cf5b56baf2f173c11708a11448 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122c489caf40f1b42a5aff4675f81ae9230_e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:29,285 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122e5e8adb2eb704cc7a824887d99ca7dfa_e2eea2cf5b56baf2f173c11708a11448 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122e5e8adb2eb704cc7a824887d99ca7dfa_e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:29,286 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122ea09690e921d46838a5c31cefad8bd6f_e2eea2cf5b56baf2f173c11708a11448 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122ea09690e921d46838a5c31cefad8bd6f_e2eea2cf5b56baf2f173c11708a11448 2024-11-22T18:56:29,287 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-22T18:56:29,289 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=68, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T18:56:29,291 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-22T18:56:29,293 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-22T18:56:29,294 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=68, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T18:56:29,294 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-22T18:56:29,294 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732301789294"}]},"ts":"9223372036854775807"} 2024-11-22T18:56:29,296 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-22T18:56:29,296 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => e2eea2cf5b56baf2f173c11708a11448, NAME => 'TestAcidGuarantees,,1732301759896.e2eea2cf5b56baf2f173c11708a11448.', STARTKEY => '', ENDKEY => ''}] 2024-11-22T18:56:29,296 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-22T18:56:29,296 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732301789296"}]},"ts":"9223372036854775807"} 2024-11-22T18:56:29,298 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-22T18:56:29,298 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=68, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T18:56:29,300 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 63 msec 2024-11-22T18:56:29,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-22T18:56:29,346 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-22T18:56:29,346 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-22T18:56:29,355 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=240 (was 237) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-258122325_22 at /127.0.0.1:59634 [Waiting for operation #209] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/001b4aa4-1a7f-6591-3cef-5770bc0658f1/cluster_d9590100-86d5-54e6-c888-90717c40c23c/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/001b4aa4-1a7f-6591-3cef-5770bc0658f1/cluster_d9590100-86d5-54e6-c888-90717c40c23c/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1775321642_22 at /127.0.0.1:40326 [Waiting for operation #258] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=451 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=317 (was 313) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7945 (was 8024) 2024-11-22T18:56:29,363 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=240, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=317, ProcessCount=11, AvailableMemoryMB=7945 2024-11-22T18:56:29,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-22T18:56:29,365 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T18:56:29,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] procedure2.ProcedureExecutor(1139): Stored pid=69, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-22T18:56:29,366 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=69, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-22T18:56:29,366 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:56:29,366 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 69 2024-11-22T18:56:29,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=69 2024-11-22T18:56:29,367 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=69, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-22T18:56:29,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742037_1213 (size=959) 2024-11-22T18:56:29,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=69 2024-11-22T18:56:29,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=69 2024-11-22T18:56:29,774 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875 2024-11-22T18:56:29,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742038_1214 (size=53) 2024-11-22T18:56:29,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=69 2024-11-22T18:56:30,180 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T18:56:30,180 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 2f5686bd89d51ddd0842891a5a86fd98, disabling compactions & flushes 2024-11-22T18:56:30,180 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:30,180 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:30,180 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. after waiting 0 ms 2024-11-22T18:56:30,180 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:30,180 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:30,180 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 2f5686bd89d51ddd0842891a5a86fd98: Waiting for close lock at 1732301790180Disabling compacts and flushes for region at 1732301790180Disabling writes for close at 1732301790180Writing region close event to WAL at 1732301790180Closed at 1732301790180 2024-11-22T18:56:30,181 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=69, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-22T18:56:30,182 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732301790181"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732301790181"}]},"ts":"1732301790181"} 2024-11-22T18:56:30,183 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-22T18:56:30,184 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=69, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-22T18:56:30,184 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732301790184"}]},"ts":"1732301790184"} 2024-11-22T18:56:30,185 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-22T18:56:30,185 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=70, ppid=69, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2f5686bd89d51ddd0842891a5a86fd98, ASSIGN}] 2024-11-22T18:56:30,186 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=69, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2f5686bd89d51ddd0842891a5a86fd98, ASSIGN 2024-11-22T18:56:30,187 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=70, ppid=69, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2f5686bd89d51ddd0842891a5a86fd98, ASSIGN; state=OFFLINE, location=d79ba0c344fb,43593,1732301724173; forceNewPlan=false, retain=false 2024-11-22T18:56:30,337 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=2f5686bd89d51ddd0842891a5a86fd98, regionState=OPENING, regionLocation=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:30,339 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=70, ppid=69, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2f5686bd89d51ddd0842891a5a86fd98, ASSIGN because future has completed 2024-11-22T18:56:30,339 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173}] 2024-11-22T18:56:30,493 INFO [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:30,494 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(7752): Opening region: {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} 2024-11-22T18:56:30,494 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 2f5686bd89d51ddd0842891a5a86fd98 2024-11-22T18:56:30,494 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T18:56:30,494 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(7794): checking encryption for 2f5686bd89d51ddd0842891a5a86fd98 2024-11-22T18:56:30,494 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(7797): checking classloading for 2f5686bd89d51ddd0842891a5a86fd98 2024-11-22T18:56:30,495 INFO [StoreOpener-2f5686bd89d51ddd0842891a5a86fd98-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 2f5686bd89d51ddd0842891a5a86fd98 2024-11-22T18:56:30,496 INFO [StoreOpener-2f5686bd89d51ddd0842891a5a86fd98-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2f5686bd89d51ddd0842891a5a86fd98 columnFamilyName A 2024-11-22T18:56:30,496 DEBUG [StoreOpener-2f5686bd89d51ddd0842891a5a86fd98-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:56:30,497 INFO [StoreOpener-2f5686bd89d51ddd0842891a5a86fd98-1 {}] regionserver.HStore(327): Store=2f5686bd89d51ddd0842891a5a86fd98/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T18:56:30,497 INFO [StoreOpener-2f5686bd89d51ddd0842891a5a86fd98-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 2f5686bd89d51ddd0842891a5a86fd98 2024-11-22T18:56:30,498 INFO [StoreOpener-2f5686bd89d51ddd0842891a5a86fd98-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2f5686bd89d51ddd0842891a5a86fd98 columnFamilyName B 2024-11-22T18:56:30,498 DEBUG [StoreOpener-2f5686bd89d51ddd0842891a5a86fd98-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:56:30,498 INFO [StoreOpener-2f5686bd89d51ddd0842891a5a86fd98-1 {}] regionserver.HStore(327): Store=2f5686bd89d51ddd0842891a5a86fd98/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T18:56:30,498 INFO [StoreOpener-2f5686bd89d51ddd0842891a5a86fd98-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 2f5686bd89d51ddd0842891a5a86fd98 2024-11-22T18:56:30,499 INFO [StoreOpener-2f5686bd89d51ddd0842891a5a86fd98-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2f5686bd89d51ddd0842891a5a86fd98 columnFamilyName C 2024-11-22T18:56:30,499 DEBUG [StoreOpener-2f5686bd89d51ddd0842891a5a86fd98-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:56:30,500 INFO [StoreOpener-2f5686bd89d51ddd0842891a5a86fd98-1 {}] regionserver.HStore(327): Store=2f5686bd89d51ddd0842891a5a86fd98/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T18:56:30,500 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(1038): replaying wal for 2f5686bd89d51ddd0842891a5a86fd98 2024-11-22T18:56:30,500 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98 2024-11-22T18:56:30,501 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98 2024-11-22T18:56:30,501 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(1048): stopping wal replay for 2f5686bd89d51ddd0842891a5a86fd98 2024-11-22T18:56:30,501 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(1060): Cleaning up temporary data for 2f5686bd89d51ddd0842891a5a86fd98 2024-11-22T18:56:30,502 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-22T18:56:30,503 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(1093): writing seq id for 2f5686bd89d51ddd0842891a5a86fd98 2024-11-22T18:56:30,504 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T18:56:30,504 INFO [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(1114): Opened 2f5686bd89d51ddd0842891a5a86fd98; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70850068, jitterRate=0.05574828386306763}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-22T18:56:30,505 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2f5686bd89d51ddd0842891a5a86fd98 2024-11-22T18:56:30,505 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegion(1006): Region open journal for 2f5686bd89d51ddd0842891a5a86fd98: Running coprocessor pre-open hook at 1732301790494Writing region info on filesystem at 1732301790494Initializing all the Stores at 1732301790495 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732301790495Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732301790495Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732301790495Cleaning up temporary data from old regions at 1732301790501 (+6 ms)Running coprocessor post-open hooks at 1732301790505 (+4 ms)Region opened successfully at 1732301790505 2024-11-22T18:56:30,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=69 2024-11-22T18:56:30,506 INFO [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., pid=71, masterSystemTime=1732301790490 2024-11-22T18:56:30,507 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:30,507 INFO [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=71}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:30,508 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=2f5686bd89d51ddd0842891a5a86fd98, regionState=OPEN, openSeqNum=2, regionLocation=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:30,509 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=71, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 because future has completed 2024-11-22T18:56:30,511 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-22T18:56:30,512 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; OpenRegionProcedure 2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 in 171 msec 2024-11-22T18:56:30,513 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=70, resume processing ppid=69 2024-11-22T18:56:30,513 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, ppid=69, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2f5686bd89d51ddd0842891a5a86fd98, ASSIGN in 327 msec 2024-11-22T18:56:30,514 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=69, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-22T18:56:30,514 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732301790514"}]},"ts":"1732301790514"} 2024-11-22T18:56:30,515 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-22T18:56:30,516 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=69, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-22T18:56:30,517 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1510 sec 2024-11-22T18:56:31,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=69 2024-11-22T18:56:31,525 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-22T18:56:31,526 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61a754db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:56:31,526 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d79ba0c344fb,44109,-1 for getting cluster id 2024-11-22T18:56:31,527 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T18:56:31,527 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '058c0dbe-7061-419a-804d-605a1c1b6341' 2024-11-22T18:56:31,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T18:56:31,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "058c0dbe-7061-419a-804d-605a1c1b6341" 2024-11-22T18:56:31,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49c82e9e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:56:31,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d79ba0c344fb,44109,-1] 2024-11-22T18:56:31,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T18:56:31,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:56:31,529 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44916, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T18:56:31,530 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d9b3121, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:56:31,530 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T18:56:31,531 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d79ba0c344fb,43593,1732301724173, seqNum=-1] 2024-11-22T18:56:31,531 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T18:56:31,532 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53214, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T18:56:31,532 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-22T18:56:31,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is d79ba0c344fb,44109,1732301723398 2024-11-22T18:56:31,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5c38370a 2024-11-22T18:56:31,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-22T18:56:31,534 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44928, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-22T18:56:31,535 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50502613, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:56:31,535 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d79ba0c344fb,44109,-1 for getting cluster id 2024-11-22T18:56:31,535 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T18:56:31,536 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '058c0dbe-7061-419a-804d-605a1c1b6341' 2024-11-22T18:56:31,536 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T18:56:31,536 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "058c0dbe-7061-419a-804d-605a1c1b6341" 2024-11-22T18:56:31,536 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e077fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:56:31,536 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d79ba0c344fb,44109,-1] 2024-11-22T18:56:31,536 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T18:56:31,536 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:56:31,537 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44938, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T18:56:31,537 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9513899, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:56:31,538 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@554689fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:56:31,538 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d79ba0c344fb,44109,-1 for getting cluster id 2024-11-22T18:56:31,538 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T18:56:31,539 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '058c0dbe-7061-419a-804d-605a1c1b6341' 2024-11-22T18:56:31,539 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T18:56:31,539 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "058c0dbe-7061-419a-804d-605a1c1b6341" 2024-11-22T18:56:31,539 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fbbb33b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:56:31,539 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d79ba0c344fb,44109,-1] 2024-11-22T18:56:31,539 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T18:56:31,539 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:56:31,540 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44950, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T18:56:31,540 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61bc998c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:56:31,541 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72325ab8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:56:31,541 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d79ba0c344fb,44109,-1 for getting cluster id 2024-11-22T18:56:31,541 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T18:56:31,541 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '058c0dbe-7061-419a-804d-605a1c1b6341' 2024-11-22T18:56:31,542 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T18:56:31,542 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "058c0dbe-7061-419a-804d-605a1c1b6341" 2024-11-22T18:56:31,542 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@726c5bfd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:56:31,542 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d79ba0c344fb,44109,-1] 2024-11-22T18:56:31,542 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T18:56:31,542 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:56:31,543 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44968, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T18:56:31,543 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61e8b8f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:56:31,544 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6686bc48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:56:31,544 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d79ba0c344fb,44109,-1 for getting cluster id 2024-11-22T18:56:31,544 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T18:56:31,544 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '058c0dbe-7061-419a-804d-605a1c1b6341' 2024-11-22T18:56:31,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T18:56:31,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "058c0dbe-7061-419a-804d-605a1c1b6341" 2024-11-22T18:56:31,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32ce4a58, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:56:31,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d79ba0c344fb,44109,-1] 2024-11-22T18:56:31,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T18:56:31,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:56:31,545 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44988, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T18:56:31,546 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a269230, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:56:31,546 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d70004e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:56:31,546 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d79ba0c344fb,44109,-1 for getting cluster id 2024-11-22T18:56:31,546 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T18:56:31,547 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '058c0dbe-7061-419a-804d-605a1c1b6341' 2024-11-22T18:56:31,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T18:56:31,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "058c0dbe-7061-419a-804d-605a1c1b6341" 2024-11-22T18:56:31,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c0b56e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:56:31,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d79ba0c344fb,44109,-1] 2024-11-22T18:56:31,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T18:56:31,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:56:31,548 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44992, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T18:56:31,549 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ec3c149, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:56:31,549 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@675e61a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:56:31,549 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d79ba0c344fb,44109,-1 for getting cluster id 2024-11-22T18:56:31,549 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T18:56:31,550 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '058c0dbe-7061-419a-804d-605a1c1b6341' 2024-11-22T18:56:31,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T18:56:31,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "058c0dbe-7061-419a-804d-605a1c1b6341" 2024-11-22T18:56:31,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40e44a1d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:56:31,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d79ba0c344fb,44109,-1] 2024-11-22T18:56:31,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T18:56:31,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:56:31,551 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45020, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T18:56:31,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dc6cb6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:56:31,552 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b9635b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:56:31,552 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d79ba0c344fb,44109,-1 for getting cluster id 2024-11-22T18:56:31,552 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T18:56:31,553 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '058c0dbe-7061-419a-804d-605a1c1b6341' 2024-11-22T18:56:31,553 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T18:56:31,553 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "058c0dbe-7061-419a-804d-605a1c1b6341" 2024-11-22T18:56:31,553 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6adfa30a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:56:31,553 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d79ba0c344fb,44109,-1] 2024-11-22T18:56:31,553 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T18:56:31,553 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:56:31,554 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45048, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T18:56:31,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bcc88e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:56:31,555 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2611025, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:56:31,555 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d79ba0c344fb,44109,-1 for getting cluster id 2024-11-22T18:56:31,555 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T18:56:31,555 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '058c0dbe-7061-419a-804d-605a1c1b6341' 2024-11-22T18:56:31,555 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T18:56:31,556 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "058c0dbe-7061-419a-804d-605a1c1b6341" 2024-11-22T18:56:31,556 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40fd9bf1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:56:31,556 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d79ba0c344fb,44109,-1] 2024-11-22T18:56:31,556 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T18:56:31,556 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:56:31,556 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45068, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T18:56:31,557 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@418b3ee2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:56:31,557 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8a8ae50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:56:31,558 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d79ba0c344fb,44109,-1 for getting cluster id 2024-11-22T18:56:31,558 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T18:56:31,558 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '058c0dbe-7061-419a-804d-605a1c1b6341' 2024-11-22T18:56:31,558 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T18:56:31,558 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "058c0dbe-7061-419a-804d-605a1c1b6341" 2024-11-22T18:56:31,559 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21228b44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:56:31,559 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d79ba0c344fb,44109,-1] 2024-11-22T18:56:31,559 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T18:56:31,559 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:56:31,559 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45078, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T18:56:31,560 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57eff370, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:56:31,560 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@471771d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:56:31,560 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d79ba0c344fb,44109,-1 for getting cluster id 2024-11-22T18:56:31,560 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T18:56:31,561 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '058c0dbe-7061-419a-804d-605a1c1b6341' 2024-11-22T18:56:31,561 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T18:56:31,561 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "058c0dbe-7061-419a-804d-605a1c1b6341" 2024-11-22T18:56:31,561 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72667771, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:56:31,561 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d79ba0c344fb,44109,-1] 2024-11-22T18:56:31,561 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T18:56:31,561 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:56:31,562 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45096, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T18:56:31,562 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3eec9264, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:56:31,563 DEBUG [Thread-1017 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T18:56:31,563 DEBUG [Thread-1032 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T18:56:31,563 DEBUG [Thread-1021 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T18:56:31,563 DEBUG [Thread-1026 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T18:56:31,563 DEBUG [Thread-1030 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T18:56:31,563 DEBUG [Thread-1024 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T18:56:31,563 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T18:56:31,564 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d79ba0c344fb,43593,1732301724173, seqNum=-1] 2024-11-22T18:56:31,564 DEBUG [Thread-1013 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T18:56:31,564 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d79ba0c344fb,43593,1732301724173, seqNum=-1] 2024-11-22T18:56:31,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d79ba0c344fb,43593,1732301724173, seqNum=-1] 2024-11-22T18:56:31,564 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T18:56:31,564 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T18:56:31,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T18:56:31,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d79ba0c344fb,43593,1732301724173, seqNum=-1] 2024-11-22T18:56:31,564 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d79ba0c344fb,43593,1732301724173, seqNum=-1] 2024-11-22T18:56:31,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T18:56:31,564 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T18:56:31,565 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d79ba0c344fb,43593,1732301724173, seqNum=-1] 2024-11-22T18:56:31,565 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T18:56:31,565 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53250, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T18:56:31,565 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d79ba0c344fb,43593,1732301724173, seqNum=-1] 2024-11-22T18:56:31,565 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53230, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T18:56:31,565 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T18:56:31,565 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53260, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T18:56:31,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-22T18:56:31,566 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53228, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T18:56:31,566 DEBUG [Thread-1028 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T18:56:31,566 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53274, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T18:56:31,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-22T18:56:31,567 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53290, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T18:56:31,567 DEBUG [Thread-1015 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T18:56:31,567 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d79ba0c344fb,43593,1732301724173, seqNum=-1] 2024-11-22T18:56:31,567 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T18:56:31,567 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53306, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T18:56:31,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2] 2024-11-22T18:56:31,568 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2] 2024-11-22T18:56:31,569 DEBUG [Thread-1019 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T18:56:31,569 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d79ba0c344fb,43593,1732301724173, seqNum=-1] 2024-11-22T18:56:31,569 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T18:56:31,569 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T18:56:31,569 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53308, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T18:56:31,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2] 2024-11-22T18:56:31,569 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2] 2024-11-22T18:56:31,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2] 2024-11-22T18:56:31,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2] 2024-11-22T18:56:31,570 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T18:56:31,570 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T18:56:31,571 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53324, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T18:56:31,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2] 2024-11-22T18:56:31,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2] 2024-11-22T18:56:31,572 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d79ba0c344fb,43593,1732301724173, seqNum=-1] 2024-11-22T18:56:31,572 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T18:56:31,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2] 2024-11-22T18:56:31,574 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53340, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T18:56:31,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2] 2024-11-22T18:56:31,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on 2f5686bd89d51ddd0842891a5a86fd98 2024-11-22T18:56:31,578 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T18:56:31,578 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2f5686bd89d51ddd0842891a5a86fd98 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-22T18:56:31,607 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/06009c221f8c4d44b7023b2016014280 is 50, key is test_row_0/A:col10/1732301791576/Put/seqid=0 2024-11-22T18:56:31,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:31,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53228 deadline: 1732301851609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:31,615 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:31,615 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:31,615 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:31,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:31,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53340 deadline: 1732301851611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:31,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:31,618 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:31,618 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:31,619 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:31,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53324 deadline: 1732301851611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:31,619 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:31,619 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:31,619 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:31,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:31,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53306 deadline: 1732301851614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:31,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:31,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:31,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:31,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:31,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53250 deadline: 1732301851613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:31,620 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:31,620 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:31,620 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:31,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742039_1215 (size=12001) 2024-11-22T18:56:31,628 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/06009c221f8c4d44b7023b2016014280 2024-11-22T18:56:31,657 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/5a4c214d15854124a0a1b469ea019b16 is 50, key is test_row_0/B:col10/1732301791576/Put/seqid=0 2024-11-22T18:56:31,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742040_1216 (size=12001) 2024-11-22T18:56:31,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-22T18:56:31,723 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-22T18:56:31,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:31,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. as already flushing 2024-11-22T18:56:31,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:31,723 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:31,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:31,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:31,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:31,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53228 deadline: 1732301851725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:31,727 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:31,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:31,727 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:31,728 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:31,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53324 deadline: 1732301851725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:31,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:31,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:31,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:31,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:31,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53250 deadline: 1732301851726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:31,728 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:31,728 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:31,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:31,728 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:31,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53340 deadline: 1732301851726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:31,729 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:31,729 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:31,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:31,729 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:31,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53306 deadline: 1732301851726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:31,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:31,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:31,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:31,876 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-22T18:56:31,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:31,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. as already flushing 2024-11-22T18:56:31,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:31,876 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:31,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:31,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:31,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-22T18:56:31,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:31,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53228 deadline: 1732301851935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:31,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:31,939 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:31,939 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:31,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53324 deadline: 1732301851935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:31,939 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:31,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:31,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:31,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53340 deadline: 1732301851936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:31,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:31,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:31,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:31,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53250 deadline: 1732301851936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:31,939 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:31,939 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:31,939 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:31,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:31,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:31,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53306 deadline: 1732301851936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:31,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:31,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:31,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:31,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:31,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:32,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-22T18:56:32,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:32,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. as already flushing 2024-11-22T18:56:32,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:32,030 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:32,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:32,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:32,075 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/5a4c214d15854124a0a1b469ea019b16 2024-11-22T18:56:32,096 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/6afe73a7d389494ea0ace9014167ec72 is 50, key is test_row_0/C:col10/1732301791576/Put/seqid=0 2024-11-22T18:56:32,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742041_1217 (size=12001) 2024-11-22T18:56:32,101 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/6afe73a7d389494ea0ace9014167ec72 2024-11-22T18:56:32,122 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/06009c221f8c4d44b7023b2016014280 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/06009c221f8c4d44b7023b2016014280 2024-11-22T18:56:32,128 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/06009c221f8c4d44b7023b2016014280, entries=150, sequenceid=12, filesize=11.7 K 2024-11-22T18:56:32,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/5a4c214d15854124a0a1b469ea019b16 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/5a4c214d15854124a0a1b469ea019b16 2024-11-22T18:56:32,133 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/5a4c214d15854124a0a1b469ea019b16, entries=150, sequenceid=12, filesize=11.7 K 2024-11-22T18:56:32,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/6afe73a7d389494ea0ace9014167ec72 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/6afe73a7d389494ea0ace9014167ec72 2024-11-22T18:56:32,139 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/6afe73a7d389494ea0ace9014167ec72, entries=150, sequenceid=12, filesize=11.7 K 2024-11-22T18:56:32,140 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 2f5686bd89d51ddd0842891a5a86fd98 in 562ms, sequenceid=12, compaction requested=false 2024-11-22T18:56:32,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2f5686bd89d51ddd0842891a5a86fd98: 2024-11-22T18:56:32,183 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-22T18:56:32,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:32,183 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 2f5686bd89d51ddd0842891a5a86fd98 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-22T18:56:32,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/9d15795c7b624832b98a6871760e268d is 50, key is test_row_0/A:col10/1732301791612/Put/seqid=0 2024-11-22T18:56:32,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742042_1218 (size=12001) 2024-11-22T18:56:32,195 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/9d15795c7b624832b98a6871760e268d 2024-11-22T18:56:32,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-22T18:56:32,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/244b3346b86b49eab9a34443228e6acd is 50, key is test_row_0/B:col10/1732301791612/Put/seqid=0 2024-11-22T18:56:32,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742043_1219 (size=12001) 2024-11-22T18:56:32,221 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/244b3346b86b49eab9a34443228e6acd 2024-11-22T18:56:32,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/b61b5303ad6f4be8b4c79b0315592f30 is 50, key is test_row_0/C:col10/1732301791612/Put/seqid=0 2024-11-22T18:56:32,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742044_1220 (size=12001) 2024-11-22T18:56:32,239 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/b61b5303ad6f4be8b4c79b0315592f30 2024-11-22T18:56:32,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/9d15795c7b624832b98a6871760e268d as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/9d15795c7b624832b98a6871760e268d 2024-11-22T18:56:32,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on 2f5686bd89d51ddd0842891a5a86fd98 2024-11-22T18:56:32,249 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. as already flushing 2024-11-22T18:56:32,254 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/9d15795c7b624832b98a6871760e268d, entries=150, sequenceid=38, filesize=11.7 K 2024-11-22T18:56:32,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/244b3346b86b49eab9a34443228e6acd as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/244b3346b86b49eab9a34443228e6acd 2024-11-22T18:56:32,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:32,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53306 deadline: 1732301852253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:32,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:32,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:32,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:32,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:32,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53228 deadline: 1732301852255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:32,261 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/244b3346b86b49eab9a34443228e6acd, entries=150, sequenceid=38, filesize=11.7 K 2024-11-22T18:56:32,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:32,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:32,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:32,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/b61b5303ad6f4be8b4c79b0315592f30 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/b61b5303ad6f4be8b4c79b0315592f30 2024-11-22T18:56:32,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:32,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53324 deadline: 1732301852260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:32,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:32,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:32,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:32,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:32,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53340 deadline: 1732301852260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:32,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:32,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:32,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:32,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53250 deadline: 1732301852262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:32,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:32,266 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:32,266 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:32,266 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:32,270 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/b61b5303ad6f4be8b4c79b0315592f30, entries=150, sequenceid=38, filesize=11.7 K 2024-11-22T18:56:32,271 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=60.38 KB/61830 for 2f5686bd89d51ddd0842891a5a86fd98 in 87ms, sequenceid=38, compaction requested=false 2024-11-22T18:56:32,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 2f5686bd89d51ddd0842891a5a86fd98: 2024-11-22T18:56:32,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:32,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-22T18:56:32,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-22T18:56:32,276 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-22T18:56:32,276 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 704 msec 2024-11-22T18:56:32,281 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 715 msec 2024-11-22T18:56:32,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on 2f5686bd89d51ddd0842891a5a86fd98 2024-11-22T18:56:32,367 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T18:56:32,367 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2f5686bd89d51ddd0842891a5a86fd98 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-22T18:56:32,373 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/0b154bac2b4b4b05a5054ad4399fd0f6 is 50, key is test_row_0/A:col10/1732301792258/Put/seqid=0 2024-11-22T18:56:32,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742045_1221 (size=14341) 2024-11-22T18:56:32,378 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/0b154bac2b4b4b05a5054ad4399fd0f6 2024-11-22T18:56:32,386 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/c34952d9476d4a0d851618c24f7c431b is 50, key is test_row_0/B:col10/1732301792258/Put/seqid=0 2024-11-22T18:56:32,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742046_1222 (size=12001) 2024-11-22T18:56:32,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:32,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53306 deadline: 1732301852389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:32,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:32,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:32,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:32,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:32,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53324 deadline: 1732301852391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:32,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:32,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:32,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:32,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53340 deadline: 1732301852392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:32,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:32,396 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:32,396 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:32,396 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:32,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:32,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53228 deadline: 1732301852393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:32,398 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:32,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:32,398 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:32,398 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:32,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53250 deadline: 1732301852394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:32,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:32,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:32,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:32,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:32,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53306 deadline: 1732301852505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:32,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:32,509 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:32,509 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:32,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53340 deadline: 1732301852505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:32,509 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:32,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:32,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53324 deadline: 1732301852505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:32,509 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:32,509 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:32,509 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:32,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:32,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:32,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53228 deadline: 1732301852506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:32,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:32,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:32,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:32,510 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:32,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53250 deadline: 1732301852506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:32,510 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:32,510 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:32,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:32,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:32,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:32,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-22T18:56:32,706 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T18:56:32,706 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T18:56:32,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-22T18:56:32,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-22T18:56:32,708 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T18:56:32,708 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T18:56:32,708 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T18:56:32,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:32,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53306 deadline: 1732301852715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:32,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:32,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:32,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:32,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:32,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53340 deadline: 1732301852716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:32,720 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:32,720 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:32,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:32,720 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:32,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53324 deadline: 1732301852716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:32,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:32,720 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:32,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:32,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53228 deadline: 1732301852716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:32,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:32,721 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:32,721 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:32,721 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:32,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:32,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53250 deadline: 1732301852716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:32,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:32,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:32,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:32,790 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/c34952d9476d4a0d851618c24f7c431b 2024-11-22T18:56:32,798 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/77f983792edd4b2c94aadbba1d826681 is 50, key is test_row_0/C:col10/1732301792258/Put/seqid=0 2024-11-22T18:56:32,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742047_1223 (size=12001) 2024-11-22T18:56:32,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-22T18:56:32,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-22T18:56:32,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:32,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. as already flushing 2024-11-22T18:56:32,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:32,861 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:32,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:32,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:33,014 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-22T18:56:33,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:33,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. as already flushing 2024-11-22T18:56:33,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:33,014 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:33,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:33,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:33,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-22T18:56:33,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:33,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53306 deadline: 1732301853025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:33,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:33,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:33,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:33,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53340 deadline: 1732301853025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:33,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:33,029 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:33,029 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:33,029 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:33,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:33,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53324 deadline: 1732301853026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:33,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:33,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:33,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:33,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:33,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:33,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53228 deadline: 1732301853026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:33,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53250 deadline: 1732301853026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:33,030 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:33,030 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:33,030 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:33,030 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:33,030 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:33,030 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:33,167 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-22T18:56:33,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:33,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. as already flushing 2024-11-22T18:56:33,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:33,167 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:33,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:33,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:33,203 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/77f983792edd4b2c94aadbba1d826681 2024-11-22T18:56:33,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/0b154bac2b4b4b05a5054ad4399fd0f6 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/0b154bac2b4b4b05a5054ad4399fd0f6 2024-11-22T18:56:33,212 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/0b154bac2b4b4b05a5054ad4399fd0f6, entries=200, sequenceid=51, filesize=14.0 K 2024-11-22T18:56:33,213 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/c34952d9476d4a0d851618c24f7c431b as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/c34952d9476d4a0d851618c24f7c431b 2024-11-22T18:56:33,217 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/c34952d9476d4a0d851618c24f7c431b, entries=150, sequenceid=51, filesize=11.7 K 2024-11-22T18:56:33,218 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/77f983792edd4b2c94aadbba1d826681 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/77f983792edd4b2c94aadbba1d826681 2024-11-22T18:56:33,223 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/77f983792edd4b2c94aadbba1d826681, entries=150, sequenceid=51, filesize=11.7 K 2024-11-22T18:56:33,224 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 2f5686bd89d51ddd0842891a5a86fd98 in 857ms, sequenceid=51, compaction requested=true 2024-11-22T18:56:33,224 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2f5686bd89d51ddd0842891a5a86fd98: 2024-11-22T18:56:33,225 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2f5686bd89d51ddd0842891a5a86fd98:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T18:56:33,225 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T18:56:33,225 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:56:33,225 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T18:56:33,225 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2f5686bd89d51ddd0842891a5a86fd98:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T18:56:33,225 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:56:33,225 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2f5686bd89d51ddd0842891a5a86fd98:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T18:56:33,225 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:56:33,229 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T18:56:33,229 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T18:56:33,229 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1541): 2f5686bd89d51ddd0842891a5a86fd98/B is initiating minor compaction (all files) 2024-11-22T18:56:33,229 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1541): 2f5686bd89d51ddd0842891a5a86fd98/A is initiating minor compaction (all files) 2024-11-22T18:56:33,229 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2f5686bd89d51ddd0842891a5a86fd98/B in TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:33,229 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2f5686bd89d51ddd0842891a5a86fd98/A in TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:33,229 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/5a4c214d15854124a0a1b469ea019b16, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/244b3346b86b49eab9a34443228e6acd, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/c34952d9476d4a0d851618c24f7c431b] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp, totalSize=35.2 K 2024-11-22T18:56:33,229 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/06009c221f8c4d44b7023b2016014280, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/9d15795c7b624832b98a6871760e268d, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/0b154bac2b4b4b05a5054ad4399fd0f6] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp, totalSize=37.4 K 2024-11-22T18:56:33,230 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06009c221f8c4d44b7023b2016014280, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732301791573 2024-11-22T18:56:33,230 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a4c214d15854124a0a1b469ea019b16, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732301791573 2024-11-22T18:56:33,230 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d15795c7b624832b98a6871760e268d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732301791609 2024-11-22T18:56:33,230 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 244b3346b86b49eab9a34443228e6acd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732301791609 2024-11-22T18:56:33,231 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b154bac2b4b4b05a5054ad4399fd0f6, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732301792258 2024-11-22T18:56:33,231 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting c34952d9476d4a0d851618c24f7c431b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732301792258 2024-11-22T18:56:33,239 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2f5686bd89d51ddd0842891a5a86fd98#A#compaction#183 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:56:33,239 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2f5686bd89d51ddd0842891a5a86fd98#B#compaction#184 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:56:33,240 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/7416a47e82f046219a6c76451efc1791 is 50, key is test_row_0/B:col10/1732301792258/Put/seqid=0 2024-11-22T18:56:33,240 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/31dbc548ced345a3bcb78de1b87d1830 is 50, key is test_row_0/A:col10/1732301792258/Put/seqid=0 2024-11-22T18:56:33,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742048_1224 (size=12104) 2024-11-22T18:56:33,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742049_1225 (size=12104) 2024-11-22T18:56:33,320 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-22T18:56:33,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:33,321 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 2f5686bd89d51ddd0842891a5a86fd98 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-22T18:56:33,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/1b78062a195147188cf62db5d20ac9fc is 50, key is test_row_0/A:col10/1732301792393/Put/seqid=0 2024-11-22T18:56:33,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742050_1226 (size=12001) 2024-11-22T18:56:33,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-22T18:56:33,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on 2f5686bd89d51ddd0842891a5a86fd98 2024-11-22T18:56:33,538 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. as already flushing 2024-11-22T18:56:33,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:33,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53306 deadline: 1732301853544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:33,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:33,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:33,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:33,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:33,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53340 deadline: 1732301853545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:33,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:33,552 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:33,552 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:33,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53250 deadline: 1732301853546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:33,552 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:33,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:33,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53324 deadline: 1732301853547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:33,552 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:33,552 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:33,552 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:33,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:33,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:33,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:33,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:33,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53228 deadline: 1732301853548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:33,553 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:33,553 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:33,553 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:33,656 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/7416a47e82f046219a6c76451efc1791 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/7416a47e82f046219a6c76451efc1791 2024-11-22T18:56:33,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:33,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53250 deadline: 1732301853655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:33,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:33,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:33,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53228 deadline: 1732301853656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:33,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:33,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:33,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:33,658 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:33,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53324 deadline: 1732301853656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:33,658 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:33,658 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:33,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:33,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:33,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:33,661 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/31dbc548ced345a3bcb78de1b87d1830 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/31dbc548ced345a3bcb78de1b87d1830 2024-11-22T18:56:33,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:33,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53306 deadline: 1732301853658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:33,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:33,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53340 deadline: 1732301853658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:33,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:33,662 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:33,662 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:33,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:33,662 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:33,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:33,663 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2f5686bd89d51ddd0842891a5a86fd98/B of 2f5686bd89d51ddd0842891a5a86fd98 into 7416a47e82f046219a6c76451efc1791(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:56:33,663 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2f5686bd89d51ddd0842891a5a86fd98: 2024-11-22T18:56:33,663 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., storeName=2f5686bd89d51ddd0842891a5a86fd98/B, priority=13, startTime=1732301793225; duration=0sec 2024-11-22T18:56:33,663 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:56:33,663 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2f5686bd89d51ddd0842891a5a86fd98:B 2024-11-22T18:56:33,663 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T18:56:33,664 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T18:56:33,664 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1541): 2f5686bd89d51ddd0842891a5a86fd98/C is initiating minor compaction (all files) 2024-11-22T18:56:33,664 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2f5686bd89d51ddd0842891a5a86fd98/C in TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:33,665 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/6afe73a7d389494ea0ace9014167ec72, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/b61b5303ad6f4be8b4c79b0315592f30, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/77f983792edd4b2c94aadbba1d826681] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp, totalSize=35.2 K 2024-11-22T18:56:33,665 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 6afe73a7d389494ea0ace9014167ec72, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732301791573 2024-11-22T18:56:33,665 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting b61b5303ad6f4be8b4c79b0315592f30, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732301791609 2024-11-22T18:56:33,666 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 77f983792edd4b2c94aadbba1d826681, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732301792258 2024-11-22T18:56:33,666 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2f5686bd89d51ddd0842891a5a86fd98/A of 2f5686bd89d51ddd0842891a5a86fd98 into 31dbc548ced345a3bcb78de1b87d1830(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:56:33,666 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2f5686bd89d51ddd0842891a5a86fd98: 2024-11-22T18:56:33,666 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., storeName=2f5686bd89d51ddd0842891a5a86fd98/A, priority=13, startTime=1732301793224; duration=0sec 2024-11-22T18:56:33,666 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:56:33,666 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2f5686bd89d51ddd0842891a5a86fd98:A 2024-11-22T18:56:33,674 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2f5686bd89d51ddd0842891a5a86fd98#C#compaction#186 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:56:33,674 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/9be18a8b9c6a4e3096b6c85b94e95710 is 50, key is test_row_0/C:col10/1732301792258/Put/seqid=0 2024-11-22T18:56:33,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742051_1227 (size=12104) 2024-11-22T18:56:33,731 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/1b78062a195147188cf62db5d20ac9fc 2024-11-22T18:56:33,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/d346e0e93b534140a95ee56c35bfde22 is 50, key is test_row_0/B:col10/1732301792393/Put/seqid=0 2024-11-22T18:56:33,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742052_1228 (size=12001) 2024-11-22T18:56:33,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-22T18:56:33,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:33,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53250 deadline: 1732301853865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:33,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:33,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:33,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:33,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:33,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53228 deadline: 1732301853865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:33,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:33,869 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:33,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53324 deadline: 1732301853865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:33,869 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:33,869 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:33,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:33,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:33,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53340 deadline: 1732301853866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:33,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:33,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:33,869 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:33,869 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:33,869 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:33,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:33,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53306 deadline: 1732301853866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:33,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:33,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:33,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:34,085 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/9be18a8b9c6a4e3096b6c85b94e95710 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/9be18a8b9c6a4e3096b6c85b94e95710 2024-11-22T18:56:34,090 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2f5686bd89d51ddd0842891a5a86fd98/C of 2f5686bd89d51ddd0842891a5a86fd98 into 9be18a8b9c6a4e3096b6c85b94e95710(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:56:34,090 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2f5686bd89d51ddd0842891a5a86fd98: 2024-11-22T18:56:34,090 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., storeName=2f5686bd89d51ddd0842891a5a86fd98/C, priority=13, startTime=1732301793225; duration=0sec 2024-11-22T18:56:34,090 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:56:34,090 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2f5686bd89d51ddd0842891a5a86fd98:C 2024-11-22T18:56:34,144 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/d346e0e93b534140a95ee56c35bfde22 2024-11-22T18:56:34,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/1d7c60bceb0741d7b4fb1646c7f73e32 is 50, key is test_row_0/C:col10/1732301792393/Put/seqid=0 2024-11-22T18:56:34,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742053_1229 (size=12001) 2024-11-22T18:56:34,159 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/1d7c60bceb0741d7b4fb1646c7f73e32 2024-11-22T18:56:34,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/1b78062a195147188cf62db5d20ac9fc as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/1b78062a195147188cf62db5d20ac9fc 2024-11-22T18:56:34,168 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/1b78062a195147188cf62db5d20ac9fc, entries=150, sequenceid=74, filesize=11.7 K 2024-11-22T18:56:34,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/d346e0e93b534140a95ee56c35bfde22 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/d346e0e93b534140a95ee56c35bfde22 2024-11-22T18:56:34,173 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/d346e0e93b534140a95ee56c35bfde22, entries=150, sequenceid=74, filesize=11.7 K 2024-11-22T18:56:34,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/1d7c60bceb0741d7b4fb1646c7f73e32 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/1d7c60bceb0741d7b4fb1646c7f73e32 2024-11-22T18:56:34,178 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/1d7c60bceb0741d7b4fb1646c7f73e32, entries=150, sequenceid=74, filesize=11.7 K 2024-11-22T18:56:34,179 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 2f5686bd89d51ddd0842891a5a86fd98 in 858ms, sequenceid=74, compaction requested=false 2024-11-22T18:56:34,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 2f5686bd89d51ddd0842891a5a86fd98: 2024-11-22T18:56:34,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:34,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-22T18:56:34,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on 2f5686bd89d51ddd0842891a5a86fd98 2024-11-22T18:56:34,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-22T18:56:34,179 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T18:56:34,179 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2f5686bd89d51ddd0842891a5a86fd98 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-22T18:56:34,182 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-22T18:56:34,182 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4720 sec 2024-11-22T18:56:34,184 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/a17ebd37affb40f1b3b2a8cb4d8681cd is 50, key is test_row_0/A:col10/1732301793545/Put/seqid=0 2024-11-22T18:56:34,184 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.4770 sec 2024-11-22T18:56:34,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742054_1230 (size=14341) 2024-11-22T18:56:34,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:34,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53324 deadline: 1732301854232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:34,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:34,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53250 deadline: 1732301854232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:34,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:34,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:34,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:34,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:34,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:34,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:34,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:34,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53340 deadline: 1732301854233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:34,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:34,240 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:34,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53306 deadline: 1732301854234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:34,240 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:34,240 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:34,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:34,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53228 deadline: 1732301854236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:34,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:34,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:34,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:34,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:34,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:34,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:34,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:34,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53324 deadline: 1732301854346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:34,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:34,349 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:34,349 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:34,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53250 deadline: 1732301854346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:34,349 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:34,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:34,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:34,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:34,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:34,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53340 deadline: 1732301854346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:34,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:34,349 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:34,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53228 deadline: 1732301854346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:34,349 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:34,349 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:34,349 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:34,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:34,350 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:34,350 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:34,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53306 deadline: 1732301854346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:34,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:34,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:34,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:34,498 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-22T18:56:34,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:34,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53324 deadline: 1732301854556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:34,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:34,559 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:34,559 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:34,559 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:34,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53250 deadline: 1732301854556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:34,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:34,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:34,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:34,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:34,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53340 deadline: 1732301854556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:34,560 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:34,560 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:34,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:34,560 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:34,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53228 deadline: 1732301854556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:34,560 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:34,560 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:34,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:34,560 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:34,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53306 deadline: 1732301854556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:34,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:34,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:34,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:34,588 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/a17ebd37affb40f1b3b2a8cb4d8681cd 2024-11-22T18:56:34,596 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/1e95f46f581d445ab625c7a1d3e23f69 is 50, key is test_row_0/B:col10/1732301793545/Put/seqid=0 2024-11-22T18:56:34,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742055_1231 (size=12001) 2024-11-22T18:56:34,603 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/1e95f46f581d445ab625c7a1d3e23f69 2024-11-22T18:56:34,611 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/9e9955c5798b4d8bb378a9a53a1972c2 is 50, key is test_row_0/C:col10/1732301793545/Put/seqid=0 2024-11-22T18:56:34,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742056_1232 (size=12001) 2024-11-22T18:56:34,619 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/9e9955c5798b4d8bb378a9a53a1972c2 2024-11-22T18:56:34,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/a17ebd37affb40f1b3b2a8cb4d8681cd as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/a17ebd37affb40f1b3b2a8cb4d8681cd 2024-11-22T18:56:34,629 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/a17ebd37affb40f1b3b2a8cb4d8681cd, entries=200, sequenceid=91, filesize=14.0 K 2024-11-22T18:56:34,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/1e95f46f581d445ab625c7a1d3e23f69 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/1e95f46f581d445ab625c7a1d3e23f69 2024-11-22T18:56:34,634 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/1e95f46f581d445ab625c7a1d3e23f69, entries=150, sequenceid=91, filesize=11.7 K 2024-11-22T18:56:34,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/9e9955c5798b4d8bb378a9a53a1972c2 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/9e9955c5798b4d8bb378a9a53a1972c2 2024-11-22T18:56:34,640 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/9e9955c5798b4d8bb378a9a53a1972c2, entries=150, sequenceid=91, filesize=11.7 K 2024-11-22T18:56:34,641 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 2f5686bd89d51ddd0842891a5a86fd98 in 462ms, sequenceid=91, compaction requested=true 2024-11-22T18:56:34,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2f5686bd89d51ddd0842891a5a86fd98: 2024-11-22T18:56:34,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2f5686bd89d51ddd0842891a5a86fd98:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T18:56:34,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:56:34,641 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T18:56:34,641 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T18:56:34,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2f5686bd89d51ddd0842891a5a86fd98:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T18:56:34,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:56:34,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2f5686bd89d51ddd0842891a5a86fd98:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T18:56:34,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:56:34,643 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T18:56:34,643 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T18:56:34,643 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1541): 2f5686bd89d51ddd0842891a5a86fd98/B is initiating minor compaction (all files) 2024-11-22T18:56:34,643 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1541): 2f5686bd89d51ddd0842891a5a86fd98/A is initiating minor compaction (all files) 2024-11-22T18:56:34,643 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2f5686bd89d51ddd0842891a5a86fd98/B in TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:34,643 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2f5686bd89d51ddd0842891a5a86fd98/A in TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:34,643 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/7416a47e82f046219a6c76451efc1791, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/d346e0e93b534140a95ee56c35bfde22, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/1e95f46f581d445ab625c7a1d3e23f69] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp, totalSize=35.3 K 2024-11-22T18:56:34,643 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/31dbc548ced345a3bcb78de1b87d1830, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/1b78062a195147188cf62db5d20ac9fc, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/a17ebd37affb40f1b3b2a8cb4d8681cd] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp, totalSize=37.5 K 2024-11-22T18:56:34,644 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 7416a47e82f046219a6c76451efc1791, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732301792258 2024-11-22T18:56:34,644 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 31dbc548ced345a3bcb78de1b87d1830, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732301792258 2024-11-22T18:56:34,645 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting d346e0e93b534140a95ee56c35bfde22, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1732301792383 2024-11-22T18:56:34,645 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b78062a195147188cf62db5d20ac9fc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1732301792383 2024-11-22T18:56:34,645 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e95f46f581d445ab625c7a1d3e23f69, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732301793544 2024-11-22T18:56:34,645 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting a17ebd37affb40f1b3b2a8cb4d8681cd, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732301793544 2024-11-22T18:56:34,654 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2f5686bd89d51ddd0842891a5a86fd98#A#compaction#192 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:56:34,655 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/15a3680435b447abb5ff5f23474b001e is 50, key is test_row_0/A:col10/1732301793545/Put/seqid=0 2024-11-22T18:56:34,658 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2f5686bd89d51ddd0842891a5a86fd98#B#compaction#193 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:56:34,659 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/c21e1337b0e64a409a637c7f640f880f is 50, key is test_row_0/B:col10/1732301793545/Put/seqid=0 2024-11-22T18:56:34,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742057_1233 (size=12207) 2024-11-22T18:56:34,686 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/15a3680435b447abb5ff5f23474b001e as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/15a3680435b447abb5ff5f23474b001e 2024-11-22T18:56:34,692 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2f5686bd89d51ddd0842891a5a86fd98/A of 2f5686bd89d51ddd0842891a5a86fd98 into 15a3680435b447abb5ff5f23474b001e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:56:34,693 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2f5686bd89d51ddd0842891a5a86fd98: 2024-11-22T18:56:34,693 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., storeName=2f5686bd89d51ddd0842891a5a86fd98/A, priority=13, startTime=1732301794641; duration=0sec 2024-11-22T18:56:34,693 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:56:34,693 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2f5686bd89d51ddd0842891a5a86fd98:A 2024-11-22T18:56:34,693 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T18:56:34,694 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T18:56:34,694 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1541): 2f5686bd89d51ddd0842891a5a86fd98/C is initiating minor compaction (all files) 2024-11-22T18:56:34,694 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2f5686bd89d51ddd0842891a5a86fd98/C in TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:34,695 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/9be18a8b9c6a4e3096b6c85b94e95710, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/1d7c60bceb0741d7b4fb1646c7f73e32, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/9e9955c5798b4d8bb378a9a53a1972c2] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp, totalSize=35.3 K 2024-11-22T18:56:34,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742058_1234 (size=12207) 2024-11-22T18:56:34,695 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9be18a8b9c6a4e3096b6c85b94e95710, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732301792258 2024-11-22T18:56:34,696 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d7c60bceb0741d7b4fb1646c7f73e32, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1732301792383 2024-11-22T18:56:34,696 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e9955c5798b4d8bb378a9a53a1972c2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732301793544 2024-11-22T18:56:34,702 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2f5686bd89d51ddd0842891a5a86fd98#C#compaction#194 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:56:34,703 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/df8f157cc2904766a7fc2ba3e4894f43 is 50, key is test_row_0/C:col10/1732301793545/Put/seqid=0 2024-11-22T18:56:34,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742059_1235 (size=12207) 2024-11-22T18:56:34,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-22T18:56:34,856 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T18:56:34,856 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T18:56:34,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-22T18:56:34,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-22T18:56:34,858 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T18:56:34,859 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T18:56:34,859 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T18:56:34,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on 2f5686bd89d51ddd0842891a5a86fd98 2024-11-22T18:56:34,868 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T18:56:34,868 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2f5686bd89d51ddd0842891a5a86fd98 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-22T18:56:34,874 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/9326c308691e4be5a5f7f1dd0612e52e is 50, key is test_row_0/A:col10/1732301794868/Put/seqid=0 2024-11-22T18:56:34,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:34,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742060_1236 (size=14341) 2024-11-22T18:56:34,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53250 deadline: 1732301854875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:34,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:34,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:34,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53228 deadline: 1732301854876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:34,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:34,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:34,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:34,881 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:34,881 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:34,881 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:34,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53340 deadline: 1732301854877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:34,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:34,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:34,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:34,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53306 deadline: 1732301854878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:34,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:34,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:34,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:34,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:34,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:34,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53324 deadline: 1732301854880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:34,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:34,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:34,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:34,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-22T18:56:34,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:34,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53250 deadline: 1732301854985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:34,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:34,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:34,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:34,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:34,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53228 deadline: 1732301854985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:34,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:34,989 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:34,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53340 deadline: 1732301854985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:34,989 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:34,989 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:34,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:34,990 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:34,990 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:34,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53306 deadline: 1732301854985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:34,990 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:34,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:34,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:34,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:34,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:34,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53324 deadline: 1732301854996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:34,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:34,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:34,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:35,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-22T18:56:35,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:35,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. as already flushing 2024-11-22T18:56:35,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:35,012 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:35,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:35,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:35,101 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/c21e1337b0e64a409a637c7f640f880f as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/c21e1337b0e64a409a637c7f640f880f 2024-11-22T18:56:35,106 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2f5686bd89d51ddd0842891a5a86fd98/B of 2f5686bd89d51ddd0842891a5a86fd98 into c21e1337b0e64a409a637c7f640f880f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:56:35,106 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2f5686bd89d51ddd0842891a5a86fd98: 2024-11-22T18:56:35,106 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., storeName=2f5686bd89d51ddd0842891a5a86fd98/B, priority=13, startTime=1732301794641; duration=0sec 2024-11-22T18:56:35,107 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:56:35,107 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2f5686bd89d51ddd0842891a5a86fd98:B 2024-11-22T18:56:35,113 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/df8f157cc2904766a7fc2ba3e4894f43 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/df8f157cc2904766a7fc2ba3e4894f43 2024-11-22T18:56:35,117 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2f5686bd89d51ddd0842891a5a86fd98/C of 2f5686bd89d51ddd0842891a5a86fd98 into df8f157cc2904766a7fc2ba3e4894f43(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:56:35,117 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2f5686bd89d51ddd0842891a5a86fd98: 2024-11-22T18:56:35,117 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., storeName=2f5686bd89d51ddd0842891a5a86fd98/C, priority=13, startTime=1732301794642; duration=0sec 2024-11-22T18:56:35,117 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:56:35,117 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2f5686bd89d51ddd0842891a5a86fd98:C 2024-11-22T18:56:35,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-22T18:56:35,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:35,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. as already flushing 2024-11-22T18:56:35,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:35,165 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:35,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:35,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:35,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-22T18:56:35,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:35,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53250 deadline: 1732301855195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:35,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:35,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:35,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:35,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53228 deadline: 1732301855196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:35,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:35,198 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:35,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:35,198 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:35,198 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:35,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53340 deadline: 1732301855196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:35,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:35,198 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:35,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53306 deadline: 1732301855196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:35,198 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:35,198 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:35,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:35,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:35,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:35,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:35,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53324 deadline: 1732301855205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:35,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:35,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:35,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:35,280 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/9326c308691e4be5a5f7f1dd0612e52e 2024-11-22T18:56:35,289 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/f2745634300e44718cccd86844308c12 is 50, key is test_row_0/B:col10/1732301794868/Put/seqid=0 2024-11-22T18:56:35,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742061_1237 (size=12001) 2024-11-22T18:56:35,318 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-22T18:56:35,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:35,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. as already flushing 2024-11-22T18:56:35,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:35,319 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:35,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:35,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:35,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-22T18:56:35,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:35,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. as already flushing 2024-11-22T18:56:35,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:35,472 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:35,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:35,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:35,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-22T18:56:35,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:35,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53250 deadline: 1732301855505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:35,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:35,508 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:35,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53228 deadline: 1732301855505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:35,508 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:35,508 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:35,508 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:35,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:35,508 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:35,508 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:35,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53340 deadline: 1732301855505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:35,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:35,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53306 deadline: 1732301855506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:35,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:35,509 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:35,509 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:35,509 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:35,509 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:35,509 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:35,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:35,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53324 deadline: 1732301855516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:35,518 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:35,518 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:35,518 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:35,624 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-22T18:56:35,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:35,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. as already flushing 2024-11-22T18:56:35,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:35,625 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:35,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:35,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:35,696 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/f2745634300e44718cccd86844308c12 2024-11-22T18:56:35,705 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/2f1c3b2174ae4653917041db1eacf4f6 is 50, key is test_row_0/C:col10/1732301794868/Put/seqid=0 2024-11-22T18:56:35,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742062_1238 (size=12001) 2024-11-22T18:56:35,709 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/2f1c3b2174ae4653917041db1eacf4f6 2024-11-22T18:56:35,714 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/9326c308691e4be5a5f7f1dd0612e52e as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/9326c308691e4be5a5f7f1dd0612e52e 2024-11-22T18:56:35,719 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/9326c308691e4be5a5f7f1dd0612e52e, entries=200, sequenceid=116, filesize=14.0 K 2024-11-22T18:56:35,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/f2745634300e44718cccd86844308c12 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/f2745634300e44718cccd86844308c12 2024-11-22T18:56:35,724 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/f2745634300e44718cccd86844308c12, entries=150, sequenceid=116, filesize=11.7 K 2024-11-22T18:56:35,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/2f1c3b2174ae4653917041db1eacf4f6 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/2f1c3b2174ae4653917041db1eacf4f6 2024-11-22T18:56:35,730 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/2f1c3b2174ae4653917041db1eacf4f6, entries=150, sequenceid=116, filesize=11.7 K 2024-11-22T18:56:35,731 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 2f5686bd89d51ddd0842891a5a86fd98 in 862ms, sequenceid=116, compaction requested=false 2024-11-22T18:56:35,731 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2f5686bd89d51ddd0842891a5a86fd98: 2024-11-22T18:56:35,777 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-22T18:56:35,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:35,778 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 2f5686bd89d51ddd0842891a5a86fd98 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-22T18:56:35,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/cef35b07137e4d469e33dc1d7f89dbc2 is 50, key is test_row_0/A:col10/1732301794870/Put/seqid=0 2024-11-22T18:56:35,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742063_1239 (size=12001) 2024-11-22T18:56:35,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-22T18:56:36,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on 2f5686bd89d51ddd0842891a5a86fd98 2024-11-22T18:56:36,019 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. as already flushing 2024-11-22T18:56:36,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:36,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53324 deadline: 1732301856047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:36,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:36,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53250 deadline: 1732301856048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:36,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:36,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:36,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:36,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:36,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53340 deadline: 1732301856049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:36,054 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:36,054 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:36,054 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:36,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:36,054 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:36,054 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:36,054 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:36,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53228 deadline: 1732301856049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:36,054 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:36,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:36,054 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:36,054 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:36,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53306 deadline: 1732301856049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:36,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:36,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:36,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:36,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:36,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53324 deadline: 1732301856155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:36,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:36,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:36,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:36,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:36,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53250 deadline: 1732301856165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:36,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:36,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:36,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:36,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:36,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53340 deadline: 1732301856165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:36,169 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:36,169 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:36,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:36,169 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:36,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53228 deadline: 1732301856165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:36,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:36,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:36,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:36,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:36,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53306 deadline: 1732301856165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:36,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:36,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:36,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:36,190 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/cef35b07137e4d469e33dc1d7f89dbc2 2024-11-22T18:56:36,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/40002127af134b6da1d2cb67b29eb8c9 is 50, key is test_row_0/B:col10/1732301794870/Put/seqid=0 2024-11-22T18:56:36,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742064_1240 (size=12001) 2024-11-22T18:56:36,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:36,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53324 deadline: 1732301856365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:36,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:36,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:36,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:36,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:36,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53250 deadline: 1732301856375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:36,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:36,378 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:36,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53228 deadline: 1732301856376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:36,378 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:36,378 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:36,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:36,378 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:36,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53340 deadline: 1732301856376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:36,378 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:36,378 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:36,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:36,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:36,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:36,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53306 deadline: 1732301856376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:36,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:36,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:36,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:36,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:36,602 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/40002127af134b6da1d2cb67b29eb8c9 2024-11-22T18:56:36,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/85560fa7a98c452780a6942085f2900e is 50, key is test_row_0/C:col10/1732301794870/Put/seqid=0 2024-11-22T18:56:36,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742065_1241 (size=12001) 2024-11-22T18:56:36,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:36,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53324 deadline: 1732301856676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:36,680 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:36,680 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:36,680 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:36,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:36,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53250 deadline: 1732301856685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:36,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:36,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:36,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:36,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:36,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53228 deadline: 1732301856685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:36,689 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:36,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:36,689 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:36,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53340 deadline: 1732301856685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:36,689 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:36,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:36,689 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:36,689 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:36,689 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:36,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53306 deadline: 1732301856685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:36,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:36,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:36,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:37,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-22T18:56:37,015 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/85560fa7a98c452780a6942085f2900e 2024-11-22T18:56:37,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/cef35b07137e4d469e33dc1d7f89dbc2 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/cef35b07137e4d469e33dc1d7f89dbc2 2024-11-22T18:56:37,024 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/cef35b07137e4d469e33dc1d7f89dbc2, entries=150, sequenceid=130, filesize=11.7 K 2024-11-22T18:56:37,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/40002127af134b6da1d2cb67b29eb8c9 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/40002127af134b6da1d2cb67b29eb8c9 2024-11-22T18:56:37,029 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/40002127af134b6da1d2cb67b29eb8c9, entries=150, sequenceid=130, filesize=11.7 K 2024-11-22T18:56:37,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/85560fa7a98c452780a6942085f2900e as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/85560fa7a98c452780a6942085f2900e 2024-11-22T18:56:37,035 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/85560fa7a98c452780a6942085f2900e, entries=150, sequenceid=130, filesize=11.7 K 2024-11-22T18:56:37,036 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=154.31 KB/158010 for 2f5686bd89d51ddd0842891a5a86fd98 in 1258ms, sequenceid=130, compaction requested=true 2024-11-22T18:56:37,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 2f5686bd89d51ddd0842891a5a86fd98: 2024-11-22T18:56:37,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:37,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-22T18:56:37,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-22T18:56:37,039 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-22T18:56:37,039 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1780 sec 2024-11-22T18:56:37,041 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 2.1840 sec 2024-11-22T18:56:37,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on 2f5686bd89d51ddd0842891a5a86fd98 2024-11-22T18:56:37,188 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T18:56:37,188 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2f5686bd89d51ddd0842891a5a86fd98 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-22T18:56:37,194 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/1bef9a3b4b6f4f0e99928ea8055dc581 is 50, key is test_row_0/A:col10/1732301796048/Put/seqid=0 2024-11-22T18:56:37,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742066_1242 (size=14541) 2024-11-22T18:56:37,199 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/1bef9a3b4b6f4f0e99928ea8055dc581 2024-11-22T18:56:37,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:37,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53324 deadline: 1732301857198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:37,201 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:37,201 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:37,201 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:37,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:37,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53250 deadline: 1732301857199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:37,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:37,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53228 deadline: 1732301857200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:37,205 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:37,205 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:37,205 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:37,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:37,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53340 deadline: 1732301857201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:37,206 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:37,206 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:37,206 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:37,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:37,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:37,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:37,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:37,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53306 deadline: 1732301857201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:37,206 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/2cb4118a057e48bf9ad85b79f0f0227f is 50, key is test_row_0/B:col10/1732301796048/Put/seqid=0 2024-11-22T18:56:37,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:37,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:37,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:37,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742067_1243 (size=12151) 2024-11-22T18:56:37,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:37,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53324 deadline: 1732301857306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:37,308 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:37,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:37,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:37,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:37,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53250 deadline: 1732301857316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:37,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:37,318 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:37,318 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:37,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53228 deadline: 1732301857316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:37,318 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:37,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:37,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53340 deadline: 1732301857316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:37,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:37,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:37,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:37,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:37,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53306 deadline: 1732301857316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:37,319 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:37,319 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:37,319 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:37,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:37,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:37,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:37,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:37,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53324 deadline: 1732301857516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:37,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:37,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:37,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:37,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:37,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53250 deadline: 1732301857526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:37,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:37,529 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:37,529 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:37,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53228 deadline: 1732301857526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:37,529 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:37,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:37,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53340 deadline: 1732301857526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:37,529 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:37,529 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:37,529 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:37,529 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:37,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:37,529 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:37,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53306 deadline: 1732301857526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:37,529 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:37,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:37,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:37,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:37,610 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/2cb4118a057e48bf9ad85b79f0f0227f 2024-11-22T18:56:37,619 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/31340e9b41114f45a9d3d8eac7f43d1f is 50, key is test_row_0/C:col10/1732301796048/Put/seqid=0 2024-11-22T18:56:37,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742068_1244 (size=12151) 2024-11-22T18:56:37,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:37,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53324 deadline: 1732301857826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:37,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:37,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:37,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:37,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:37,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53250 deadline: 1732301857835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:37,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:37,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:37,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:37,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:37,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53228 deadline: 1732301857835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:37,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:37,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:37,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53340 deadline: 1732301857835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:37,839 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:37,839 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:37,839 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:37,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53306 deadline: 1732301857835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:37,839 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:37,839 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:37,839 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:37,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:37,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:37,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:38,024 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/31340e9b41114f45a9d3d8eac7f43d1f 2024-11-22T18:56:38,029 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/1bef9a3b4b6f4f0e99928ea8055dc581 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/1bef9a3b4b6f4f0e99928ea8055dc581 2024-11-22T18:56:38,034 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/1bef9a3b4b6f4f0e99928ea8055dc581, entries=200, sequenceid=157, filesize=14.2 K 2024-11-22T18:56:38,035 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/2cb4118a057e48bf9ad85b79f0f0227f as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/2cb4118a057e48bf9ad85b79f0f0227f 2024-11-22T18:56:38,039 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/2cb4118a057e48bf9ad85b79f0f0227f, entries=150, sequenceid=157, filesize=11.9 K 2024-11-22T18:56:38,040 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/31340e9b41114f45a9d3d8eac7f43d1f as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/31340e9b41114f45a9d3d8eac7f43d1f 2024-11-22T18:56:38,045 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/31340e9b41114f45a9d3d8eac7f43d1f, entries=150, sequenceid=157, filesize=11.9 K 2024-11-22T18:56:38,046 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 2f5686bd89d51ddd0842891a5a86fd98 in 858ms, sequenceid=157, compaction requested=true 2024-11-22T18:56:38,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2f5686bd89d51ddd0842891a5a86fd98: 2024-11-22T18:56:38,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2f5686bd89d51ddd0842891a5a86fd98:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T18:56:38,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:56:38,046 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T18:56:38,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2f5686bd89d51ddd0842891a5a86fd98:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T18:56:38,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:56:38,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2f5686bd89d51ddd0842891a5a86fd98:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T18:56:38,046 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T18:56:38,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:56:38,048 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T18:56:38,048 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T18:56:38,048 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1541): 2f5686bd89d51ddd0842891a5a86fd98/A is initiating minor compaction (all files) 2024-11-22T18:56:38,048 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1541): 2f5686bd89d51ddd0842891a5a86fd98/B is initiating minor compaction (all files) 2024-11-22T18:56:38,048 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2f5686bd89d51ddd0842891a5a86fd98/A in TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:38,048 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2f5686bd89d51ddd0842891a5a86fd98/B in TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:38,048 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/15a3680435b447abb5ff5f23474b001e, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/9326c308691e4be5a5f7f1dd0612e52e, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/cef35b07137e4d469e33dc1d7f89dbc2, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/1bef9a3b4b6f4f0e99928ea8055dc581] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp, totalSize=51.8 K 2024-11-22T18:56:38,048 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/c21e1337b0e64a409a637c7f640f880f, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/f2745634300e44718cccd86844308c12, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/40002127af134b6da1d2cb67b29eb8c9, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/2cb4118a057e48bf9ad85b79f0f0227f] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp, totalSize=47.2 K 2024-11-22T18:56:38,049 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15a3680435b447abb5ff5f23474b001e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732301793544 2024-11-22T18:56:38,049 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting c21e1337b0e64a409a637c7f640f880f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732301793544 2024-11-22T18:56:38,049 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9326c308691e4be5a5f7f1dd0612e52e, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732301794232 2024-11-22T18:56:38,049 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting f2745634300e44718cccd86844308c12, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732301794234 2024-11-22T18:56:38,049 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting cef35b07137e4d469e33dc1d7f89dbc2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732301794870 2024-11-22T18:56:38,049 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 40002127af134b6da1d2cb67b29eb8c9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732301794870 2024-11-22T18:56:38,050 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 2cb4118a057e48bf9ad85b79f0f0227f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732301796048 2024-11-22T18:56:38,050 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1bef9a3b4b6f4f0e99928ea8055dc581, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732301796038 2024-11-22T18:56:38,059 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2f5686bd89d51ddd0842891a5a86fd98#B#compaction#204 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:56:38,059 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/a465780d00df427eae97c4e4c9a70c19 is 50, key is test_row_0/B:col10/1732301796048/Put/seqid=0 2024-11-22T18:56:38,061 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2f5686bd89d51ddd0842891a5a86fd98#A#compaction#205 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:56:38,062 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/143dea4cac924dea8772b75350cf6a4f is 50, key is test_row_0/A:col10/1732301796048/Put/seqid=0 2024-11-22T18:56:38,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742070_1246 (size=12493) 2024-11-22T18:56:38,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742069_1245 (size=12493) 2024-11-22T18:56:38,084 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/a465780d00df427eae97c4e4c9a70c19 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/a465780d00df427eae97c4e4c9a70c19 2024-11-22T18:56:38,090 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2f5686bd89d51ddd0842891a5a86fd98/B of 2f5686bd89d51ddd0842891a5a86fd98 into a465780d00df427eae97c4e4c9a70c19(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:56:38,090 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2f5686bd89d51ddd0842891a5a86fd98: 2024-11-22T18:56:38,090 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., storeName=2f5686bd89d51ddd0842891a5a86fd98/B, priority=12, startTime=1732301798046; duration=0sec 2024-11-22T18:56:38,090 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:56:38,090 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2f5686bd89d51ddd0842891a5a86fd98:B 2024-11-22T18:56:38,090 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T18:56:38,092 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T18:56:38,092 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1541): 2f5686bd89d51ddd0842891a5a86fd98/C is initiating minor compaction (all files) 2024-11-22T18:56:38,092 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2f5686bd89d51ddd0842891a5a86fd98/C in TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:38,092 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/df8f157cc2904766a7fc2ba3e4894f43, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/2f1c3b2174ae4653917041db1eacf4f6, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/85560fa7a98c452780a6942085f2900e, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/31340e9b41114f45a9d3d8eac7f43d1f] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp, totalSize=47.2 K 2024-11-22T18:56:38,092 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting df8f157cc2904766a7fc2ba3e4894f43, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732301793544 2024-11-22T18:56:38,093 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f1c3b2174ae4653917041db1eacf4f6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732301794234 2024-11-22T18:56:38,093 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 85560fa7a98c452780a6942085f2900e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732301794870 2024-11-22T18:56:38,093 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 31340e9b41114f45a9d3d8eac7f43d1f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732301796048 2024-11-22T18:56:38,106 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2f5686bd89d51ddd0842891a5a86fd98#C#compaction#206 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:56:38,107 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/cdf2e731c4d5473dae37f706a44e959d is 50, key is test_row_0/C:col10/1732301796048/Put/seqid=0 2024-11-22T18:56:38,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742071_1247 (size=12493) 2024-11-22T18:56:38,118 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/cdf2e731c4d5473dae37f706a44e959d as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/cdf2e731c4d5473dae37f706a44e959d 2024-11-22T18:56:38,125 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2f5686bd89d51ddd0842891a5a86fd98/C of 2f5686bd89d51ddd0842891a5a86fd98 into cdf2e731c4d5473dae37f706a44e959d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:56:38,125 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2f5686bd89d51ddd0842891a5a86fd98: 2024-11-22T18:56:38,125 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., storeName=2f5686bd89d51ddd0842891a5a86fd98/C, priority=12, startTime=1732301798046; duration=0sec 2024-11-22T18:56:38,125 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:56:38,126 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2f5686bd89d51ddd0842891a5a86fd98:C 2024-11-22T18:56:38,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on 2f5686bd89d51ddd0842891a5a86fd98 2024-11-22T18:56:38,339 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T18:56:38,340 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2f5686bd89d51ddd0842891a5a86fd98 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-22T18:56:38,344 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/3ba2b8e78db04582a108de6e0c35c2ff is 50, key is test_row_0/A:col10/1732301797199/Put/seqid=0 2024-11-22T18:56:38,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742072_1248 (size=14541) 2024-11-22T18:56:38,353 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/3ba2b8e78db04582a108de6e0c35c2ff 2024-11-22T18:56:38,360 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/46a8f9ba1812476ca04badd77f2e1677 is 50, key is test_row_0/B:col10/1732301797199/Put/seqid=0 2024-11-22T18:56:38,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742073_1249 (size=12151) 2024-11-22T18:56:38,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:38,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53340 deadline: 1732301858369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:38,373 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:38,373 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:38,373 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:38,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:38,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53228 deadline: 1732301858371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:38,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:38,377 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:38,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53250 deadline: 1732301858371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:38,377 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:38,377 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:38,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:38,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53306 deadline: 1732301858372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:38,377 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:38,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:38,377 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:38,377 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:38,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53324 deadline: 1732301858373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:38,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:38,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:38,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:38,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:38,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:38,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:38,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:38,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53340 deadline: 1732301858476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:38,478 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/143dea4cac924dea8772b75350cf6a4f as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/143dea4cac924dea8772b75350cf6a4f 2024-11-22T18:56:38,478 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:38,478 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:38,478 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:38,483 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2f5686bd89d51ddd0842891a5a86fd98/A of 2f5686bd89d51ddd0842891a5a86fd98 into 143dea4cac924dea8772b75350cf6a4f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:56:38,483 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2f5686bd89d51ddd0842891a5a86fd98: 2024-11-22T18:56:38,484 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., storeName=2f5686bd89d51ddd0842891a5a86fd98/A, priority=12, startTime=1732301798046; duration=0sec 2024-11-22T18:56:38,484 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:56:38,484 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2f5686bd89d51ddd0842891a5a86fd98:A 2024-11-22T18:56:38,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:38,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53228 deadline: 1732301858486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:38,487 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:38,487 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:38,487 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:38,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:38,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53250 deadline: 1732301858486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:38,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:38,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53324 deadline: 1732301858486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:38,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:38,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:38,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53306 deadline: 1732301858486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:38,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:38,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:38,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:38,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:38,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:38,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:38,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:38,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:38,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:38,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53340 deadline: 1732301858686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:38,689 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:38,689 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:38,689 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:38,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:38,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53228 deadline: 1732301858695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:38,698 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:38,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:38,698 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:38,698 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:38,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53250 deadline: 1732301858695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:38,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:38,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:38,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:38,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:38,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53306 deadline: 1732301858695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:38,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:38,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53324 deadline: 1732301858696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:38,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:38,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:38,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:38,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:38,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:38,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:38,771 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/46a8f9ba1812476ca04badd77f2e1677 2024-11-22T18:56:38,781 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/271cd2a7f3684da4bfdcebc6454fbf61 is 50, key is test_row_0/C:col10/1732301797199/Put/seqid=0 2024-11-22T18:56:38,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742074_1250 (size=12151) 2024-11-22T18:56:38,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:38,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53340 deadline: 1732301858996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:38,999 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:38,999 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:38,999 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:39,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:39,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53228 deadline: 1732301859005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:39,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:39,006 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:39,007 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:39,007 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:39,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53250 deadline: 1732301859005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:39,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:39,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:39,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:39,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:39,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53306 deadline: 1732301859005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:39,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:39,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:39,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:39,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:39,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53324 deadline: 1732301859005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:39,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:39,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:39,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:39,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-22T18:56:39,016 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T18:56:39,016 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T18:56:39,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-22T18:56:39,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-22T18:56:39,019 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T18:56:39,019 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T18:56:39,020 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T18:56:39,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-22T18:56:39,172 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-22T18:56:39,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:39,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. as already flushing 2024-11-22T18:56:39,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:39,172 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:39,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:39,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:39,186 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/271cd2a7f3684da4bfdcebc6454fbf61 2024-11-22T18:56:39,191 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/3ba2b8e78db04582a108de6e0c35c2ff as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/3ba2b8e78db04582a108de6e0c35c2ff 2024-11-22T18:56:39,195 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/3ba2b8e78db04582a108de6e0c35c2ff, entries=200, sequenceid=170, filesize=14.2 K 2024-11-22T18:56:39,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/46a8f9ba1812476ca04badd77f2e1677 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/46a8f9ba1812476ca04badd77f2e1677 2024-11-22T18:56:39,199 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/46a8f9ba1812476ca04badd77f2e1677, entries=150, sequenceid=170, filesize=11.9 K 2024-11-22T18:56:39,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/271cd2a7f3684da4bfdcebc6454fbf61 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/271cd2a7f3684da4bfdcebc6454fbf61 2024-11-22T18:56:39,208 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/271cd2a7f3684da4bfdcebc6454fbf61, entries=150, sequenceid=170, filesize=11.9 K 2024-11-22T18:56:39,213 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 2f5686bd89d51ddd0842891a5a86fd98 in 874ms, sequenceid=170, compaction requested=false 2024-11-22T18:56:39,213 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2f5686bd89d51ddd0842891a5a86fd98: 2024-11-22T18:56:39,325 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-22T18:56:39,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:39,325 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 2f5686bd89d51ddd0842891a5a86fd98 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-22T18:56:39,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/ce9d5c51e33e441f891c18eec9a628d8 is 50, key is test_row_0/A:col10/1732301798370/Put/seqid=0 2024-11-22T18:56:39,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-22T18:56:39,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742075_1251 (size=12151) 2024-11-22T18:56:39,336 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/ce9d5c51e33e441f891c18eec9a628d8 2024-11-22T18:56:39,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/e5c98c63cab741459a3e42b5eb5e4aaf is 50, key is test_row_0/B:col10/1732301798370/Put/seqid=0 2024-11-22T18:56:39,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742076_1252 (size=12151) 2024-11-22T18:56:39,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on 2f5686bd89d51ddd0842891a5a86fd98 2024-11-22T18:56:39,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. as already flushing 2024-11-22T18:56:39,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:39,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53228 deadline: 1732301859520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:39,524 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:39,524 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:39,524 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:39,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:39,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53250 deadline: 1732301859521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:39,526 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:39,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:39,526 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:39,526 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:39,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53306 deadline: 1732301859521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:39,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:39,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:39,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:39,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53324 deadline: 1732301859522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:39,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:39,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:39,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:39,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53340 deadline: 1732301859522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:39,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:39,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:39,527 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:39,527 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:39,527 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:39,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:39,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53228 deadline: 1732301859635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:39,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:39,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:39,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:39,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:39,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53250 deadline: 1732301859635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:39,637 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:39,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:39,637 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:39,637 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:39,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53306 deadline: 1732301859635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:39,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:39,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:39,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:39,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:39,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53324 deadline: 1732301859635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:39,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:39,638 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:39,638 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:39,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:39,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53340 deadline: 1732301859636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:39,638 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:39,638 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:39,638 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:39,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-22T18:56:39,748 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/e5c98c63cab741459a3e42b5eb5e4aaf 2024-11-22T18:56:39,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/ac859694f8714fd9bdd1726188d2bcd3 is 50, key is test_row_0/C:col10/1732301798370/Put/seqid=0 2024-11-22T18:56:39,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742077_1253 (size=12151) 2024-11-22T18:56:39,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:39,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53228 deadline: 1732301859845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:39,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:39,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53250 deadline: 1732301859845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:39,848 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:39,848 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:39,848 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:39,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:39,848 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:39,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53306 deadline: 1732301859845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:39,848 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:39,848 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:39,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:39,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:39,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:39,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:39,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53324 deadline: 1732301859846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:39,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:39,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:39,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:39,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53340 deadline: 1732301859846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:39,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:39,849 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:39,849 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:39,849 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:40,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-22T18:56:40,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:40,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53228 deadline: 1732301860155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:40,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:40,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53250 deadline: 1732301860155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:40,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:40,158 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:40,158 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:40,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53324 deadline: 1732301860155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:40,158 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:40,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:40,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:40,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:40,158 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:40,158 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:40,158 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:40,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:40,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53340 deadline: 1732301860156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:40,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:40,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:40,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:40,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:40,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53306 deadline: 1732301860156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:40,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:40,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:40,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:40,164 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/ac859694f8714fd9bdd1726188d2bcd3 2024-11-22T18:56:40,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/ce9d5c51e33e441f891c18eec9a628d8 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/ce9d5c51e33e441f891c18eec9a628d8 2024-11-22T18:56:40,173 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/ce9d5c51e33e441f891c18eec9a628d8, entries=150, sequenceid=196, filesize=11.9 K 2024-11-22T18:56:40,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/e5c98c63cab741459a3e42b5eb5e4aaf as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/e5c98c63cab741459a3e42b5eb5e4aaf 2024-11-22T18:56:40,177 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/e5c98c63cab741459a3e42b5eb5e4aaf, entries=150, sequenceid=196, filesize=11.9 K 2024-11-22T18:56:40,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/ac859694f8714fd9bdd1726188d2bcd3 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/ac859694f8714fd9bdd1726188d2bcd3 2024-11-22T18:56:40,183 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/ac859694f8714fd9bdd1726188d2bcd3, entries=150, sequenceid=196, filesize=11.9 K 2024-11-22T18:56:40,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-22T18:56:40,183 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 2f5686bd89d51ddd0842891a5a86fd98 in 858ms, sequenceid=196, compaction requested=true 2024-11-22T18:56:40,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 2f5686bd89d51ddd0842891a5a86fd98: 2024-11-22T18:56:40,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:40,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-22T18:56:40,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-22T18:56:40,189 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-22T18:56:40,189 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1680 sec 2024-11-22T18:56:40,191 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.1740 sec 2024-11-22T18:56:40,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on 2f5686bd89d51ddd0842891a5a86fd98 2024-11-22T18:56:40,669 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T18:56:40,669 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2f5686bd89d51ddd0842891a5a86fd98 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-22T18:56:40,675 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/85df1074a5f048319e6fa8cf44927d39 is 50, key is test_row_0/A:col10/1732301800669/Put/seqid=0 2024-11-22T18:56:40,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742078_1254 (size=12147) 2024-11-22T18:56:40,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:40,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53324 deadline: 1732301860687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:40,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:40,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:40,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:40,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:40,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53340 deadline: 1732301860690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:40,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:40,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:40,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:40,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:40,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53306 deadline: 1732301860691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:40,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:40,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:40,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:40,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:40,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53228 deadline: 1732301860692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:40,696 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:40,696 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:40,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:40,696 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:40,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53250 deadline: 1732301860692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:40,697 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:40,697 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:40,697 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:40,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:40,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53324 deadline: 1732301860796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:40,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:40,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:40,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:40,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:40,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53340 deadline: 1732301860806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:40,809 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:40,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:40,809 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:40,809 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:40,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53306 deadline: 1732301860806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:40,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:40,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:40,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:40,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:40,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53228 deadline: 1732301860806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:40,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:40,810 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:40,810 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:40,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53250 deadline: 1732301860806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:40,810 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:40,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:40,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:40,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:41,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:41,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53324 deadline: 1732301861006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:41,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:41,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:41,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:41,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:41,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53340 deadline: 1732301861016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:41,019 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:41,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:41,019 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:41,019 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:41,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53306 deadline: 1732301861016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:41,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:41,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:41,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:41,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:41,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53228 deadline: 1732301861016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:41,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:41,020 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:41,020 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:41,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53250 deadline: 1732301861016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:41,020 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:41,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:41,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:41,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:41,081 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/85df1074a5f048319e6fa8cf44927d39 2024-11-22T18:56:41,090 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/bca59a43951c4cb286aebcfbf4054ca5 is 50, key is test_row_0/B:col10/1732301800669/Put/seqid=0 2024-11-22T18:56:41,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742079_1255 (size=9757) 2024-11-22T18:56:41,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-22T18:56:41,166 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T18:56:41,166 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T18:56:41,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-22T18:56:41,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-22T18:56:41,168 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T18:56:41,169 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T18:56:41,169 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T18:56:41,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-22T18:56:41,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:41,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53324 deadline: 1732301861316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:41,320 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:41,320 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:41,320 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:41,322 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-22T18:56:41,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:41,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. as already flushing 2024-11-22T18:56:41,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:41,322 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:41,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:41,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:41,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:41,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53340 deadline: 1732301861325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:41,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:41,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:41,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:41,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:41,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53306 deadline: 1732301861325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:41,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:41,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:41,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:41,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:41,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53228 deadline: 1732301861325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:41,329 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:41,329 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:41,329 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:41,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:41,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53250 deadline: 1732301861325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:41,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:41,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:41,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:41,475 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-22T18:56:41,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:41,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. as already flushing 2024-11-22T18:56:41,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:41,476 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:41,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:41,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:41,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-22T18:56:41,495 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/bca59a43951c4cb286aebcfbf4054ca5 2024-11-22T18:56:41,503 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/de180626aede4534a0662cd27b6134c9 is 50, key is test_row_0/C:col10/1732301800669/Put/seqid=0 2024-11-22T18:56:41,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742080_1256 (size=9757) 2024-11-22T18:56:41,628 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-22T18:56:41,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:41,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. as already flushing 2024-11-22T18:56:41,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:41,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:41,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:41,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:41,781 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-22T18:56:41,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:41,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. as already flushing 2024-11-22T18:56:41,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:41,781 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:41,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:41,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:41,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-22T18:56:41,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:41,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53324 deadline: 1732301861825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:41,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:41,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:41,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:41,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:41,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53340 deadline: 1732301861835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:41,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:41,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:41,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53306 deadline: 1732301861835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:41,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:41,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:41,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:41,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:41,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53228 deadline: 1732301861835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:41,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:41,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:41,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:41,839 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:41,839 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:41,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53250 deadline: 1732301861836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:41,839 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:41,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:41,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:41,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:41,911 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/de180626aede4534a0662cd27b6134c9 2024-11-22T18:56:41,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/85df1074a5f048319e6fa8cf44927d39 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/85df1074a5f048319e6fa8cf44927d39 2024-11-22T18:56:41,920 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/85df1074a5f048319e6fa8cf44927d39, entries=150, sequenceid=210, filesize=11.9 K 2024-11-22T18:56:41,921 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/bca59a43951c4cb286aebcfbf4054ca5 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/bca59a43951c4cb286aebcfbf4054ca5 2024-11-22T18:56:41,925 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/bca59a43951c4cb286aebcfbf4054ca5, entries=100, sequenceid=210, filesize=9.5 K 2024-11-22T18:56:41,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/de180626aede4534a0662cd27b6134c9 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/de180626aede4534a0662cd27b6134c9 2024-11-22T18:56:41,931 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/de180626aede4534a0662cd27b6134c9, entries=100, sequenceid=210, filesize=9.5 K 2024-11-22T18:56:41,932 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 2f5686bd89d51ddd0842891a5a86fd98 in 1263ms, sequenceid=210, compaction requested=true 2024-11-22T18:56:41,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2f5686bd89d51ddd0842891a5a86fd98: 2024-11-22T18:56:41,932 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T18:56:41,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2f5686bd89d51ddd0842891a5a86fd98:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T18:56:41,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:56:41,933 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T18:56:41,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2f5686bd89d51ddd0842891a5a86fd98:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T18:56:41,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:56:41,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2f5686bd89d51ddd0842891a5a86fd98:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T18:56:41,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:56:41,935 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51332 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T18:56:41,935 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-22T18:56:41,935 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1541): 2f5686bd89d51ddd0842891a5a86fd98/A is initiating minor compaction (all files) 2024-11-22T18:56:41,935 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2f5686bd89d51ddd0842891a5a86fd98/A in TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:41,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:41,935 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46552 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T18:56:41,935 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/143dea4cac924dea8772b75350cf6a4f, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/3ba2b8e78db04582a108de6e0c35c2ff, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/ce9d5c51e33e441f891c18eec9a628d8, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/85df1074a5f048319e6fa8cf44927d39] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp, totalSize=50.1 K 2024-11-22T18:56:41,935 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1541): 2f5686bd89d51ddd0842891a5a86fd98/B is initiating minor compaction (all files) 2024-11-22T18:56:41,935 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2f5686bd89d51ddd0842891a5a86fd98/B in TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:41,935 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing 2f5686bd89d51ddd0842891a5a86fd98 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-22T18:56:41,935 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/a465780d00df427eae97c4e4c9a70c19, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/46a8f9ba1812476ca04badd77f2e1677, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/e5c98c63cab741459a3e42b5eb5e4aaf, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/bca59a43951c4cb286aebcfbf4054ca5] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp, totalSize=45.5 K 2024-11-22T18:56:41,936 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 143dea4cac924dea8772b75350cf6a4f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732301796048 2024-11-22T18:56:41,936 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ba2b8e78db04582a108de6e0c35c2ff, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1732301797199 2024-11-22T18:56:41,936 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting a465780d00df427eae97c4e4c9a70c19, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732301796048 2024-11-22T18:56:41,937 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 46a8f9ba1812476ca04badd77f2e1677, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1732301797199 2024-11-22T18:56:41,937 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce9d5c51e33e441f891c18eec9a628d8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732301798370 2024-11-22T18:56:41,937 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting e5c98c63cab741459a3e42b5eb5e4aaf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732301798370 2024-11-22T18:56:41,937 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85df1074a5f048319e6fa8cf44927d39, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732301800667 2024-11-22T18:56:41,938 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting bca59a43951c4cb286aebcfbf4054ca5, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732301800667 2024-11-22T18:56:41,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/3d65d24a9e8b4e66a5b2dad47f74e4dd is 50, key is test_row_0/A:col10/1732301800690/Put/seqid=0 2024-11-22T18:56:41,948 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2f5686bd89d51ddd0842891a5a86fd98#B#compaction#217 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:56:41,949 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/e6af29262ff84900993340a58d392b49 is 50, key is test_row_0/B:col10/1732301800669/Put/seqid=0 2024-11-22T18:56:41,950 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2f5686bd89d51ddd0842891a5a86fd98#A#compaction#218 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:56:41,951 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/a7e10a7e98fb46c89694e1d36c900ed5 is 50, key is test_row_0/A:col10/1732301800669/Put/seqid=0 2024-11-22T18:56:41,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742082_1258 (size=12629) 2024-11-22T18:56:41,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742083_1259 (size=12629) 2024-11-22T18:56:41,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742081_1257 (size=12151) 2024-11-22T18:56:42,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-22T18:56:42,363 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/e6af29262ff84900993340a58d392b49 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/e6af29262ff84900993340a58d392b49 2024-11-22T18:56:42,365 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/a7e10a7e98fb46c89694e1d36c900ed5 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/a7e10a7e98fb46c89694e1d36c900ed5 2024-11-22T18:56:42,367 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/3d65d24a9e8b4e66a5b2dad47f74e4dd 2024-11-22T18:56:42,369 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2f5686bd89d51ddd0842891a5a86fd98/B of 2f5686bd89d51ddd0842891a5a86fd98 into e6af29262ff84900993340a58d392b49(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:56:42,369 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2f5686bd89d51ddd0842891a5a86fd98: 2024-11-22T18:56:42,369 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., storeName=2f5686bd89d51ddd0842891a5a86fd98/B, priority=12, startTime=1732301801933; duration=0sec 2024-11-22T18:56:42,369 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:56:42,370 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2f5686bd89d51ddd0842891a5a86fd98:B 2024-11-22T18:56:42,370 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T18:56:42,372 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46552 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T18:56:42,373 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1541): 2f5686bd89d51ddd0842891a5a86fd98/C is initiating minor compaction (all files) 2024-11-22T18:56:42,373 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2f5686bd89d51ddd0842891a5a86fd98/C in TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:42,373 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/cdf2e731c4d5473dae37f706a44e959d, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/271cd2a7f3684da4bfdcebc6454fbf61, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/ac859694f8714fd9bdd1726188d2bcd3, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/de180626aede4534a0662cd27b6134c9] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp, totalSize=45.5 K 2024-11-22T18:56:42,373 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting cdf2e731c4d5473dae37f706a44e959d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732301796048 2024-11-22T18:56:42,374 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 271cd2a7f3684da4bfdcebc6454fbf61, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1732301797199 2024-11-22T18:56:42,375 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2f5686bd89d51ddd0842891a5a86fd98/A of 2f5686bd89d51ddd0842891a5a86fd98 into a7e10a7e98fb46c89694e1d36c900ed5(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:56:42,375 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2f5686bd89d51ddd0842891a5a86fd98: 2024-11-22T18:56:42,375 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., storeName=2f5686bd89d51ddd0842891a5a86fd98/A, priority=12, startTime=1732301801932; duration=0sec 2024-11-22T18:56:42,375 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:56:42,375 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2f5686bd89d51ddd0842891a5a86fd98:A 2024-11-22T18:56:42,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/24263897fe244fd0b154156039cad35c is 50, key is test_row_0/B:col10/1732301800690/Put/seqid=0 2024-11-22T18:56:42,377 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting ac859694f8714fd9bdd1726188d2bcd3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732301798370 2024-11-22T18:56:42,378 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting de180626aede4534a0662cd27b6134c9, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732301800667 2024-11-22T18:56:42,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742084_1260 (size=12151) 2024-11-22T18:56:42,398 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2f5686bd89d51ddd0842891a5a86fd98#C#compaction#220 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:56:42,398 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/41dac32379f645b19829d3b92a4e960b is 50, key is test_row_0/C:col10/1732301800669/Put/seqid=0 2024-11-22T18:56:42,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742085_1261 (size=12629) 2024-11-22T18:56:42,788 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/24263897fe244fd0b154156039cad35c 2024-11-22T18:56:42,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/c996fb88b9ec4f17bbc8ec9a70eb319a is 50, key is test_row_0/C:col10/1732301800690/Put/seqid=0 2024-11-22T18:56:42,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742086_1262 (size=12151) 2024-11-22T18:56:42,833 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/41dac32379f645b19829d3b92a4e960b as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/41dac32379f645b19829d3b92a4e960b 2024-11-22T18:56:42,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on 2f5686bd89d51ddd0842891a5a86fd98 2024-11-22T18:56:42,839 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. as already flushing 2024-11-22T18:56:42,841 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2f5686bd89d51ddd0842891a5a86fd98/C of 2f5686bd89d51ddd0842891a5a86fd98 into 41dac32379f645b19829d3b92a4e960b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:56:42,841 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2f5686bd89d51ddd0842891a5a86fd98: 2024-11-22T18:56:42,841 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., storeName=2f5686bd89d51ddd0842891a5a86fd98/C, priority=12, startTime=1732301801933; duration=0sec 2024-11-22T18:56:42,841 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:56:42,841 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2f5686bd89d51ddd0842891a5a86fd98:C 2024-11-22T18:56:42,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:42,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53324 deadline: 1732301862857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:42,862 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:42,862 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:42,862 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:42,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:42,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53306 deadline: 1732301862859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:42,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:42,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:42,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:42,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:42,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:42,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53340 deadline: 1732301862860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:42,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53228 deadline: 1732301862860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:42,865 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:42,865 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:42,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:42,865 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:42,865 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:42,865 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:42,865 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:42,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53250 deadline: 1732301862861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:42,866 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:42,866 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:42,866 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:42,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:42,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53324 deadline: 1732301862965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:42,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:42,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:42,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:42,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:42,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53306 deadline: 1732301862976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:42,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:42,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:42,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:42,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:42,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53340 deadline: 1732301862976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:42,979 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:42,979 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:42,979 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:42,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:42,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53228 deadline: 1732301862976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:42,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:42,980 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:42,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53250 deadline: 1732301862976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:42,980 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:42,980 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:42,980 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:42,980 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:42,980 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:43,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:43,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53324 deadline: 1732301863176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:43,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:43,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:43,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:43,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:43,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53306 deadline: 1732301863186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:43,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:43,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:43,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53340 deadline: 1732301863186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:43,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:43,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:43,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:43,189 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:43,189 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:43,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53228 deadline: 1732301863186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:43,189 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:43,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:43,189 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:43,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53250 deadline: 1732301863186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:43,189 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:43,189 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:43,189 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:43,190 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:43,190 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:43,222 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/c996fb88b9ec4f17bbc8ec9a70eb319a 2024-11-22T18:56:43,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/3d65d24a9e8b4e66a5b2dad47f74e4dd as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/3d65d24a9e8b4e66a5b2dad47f74e4dd 2024-11-22T18:56:43,242 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/3d65d24a9e8b4e66a5b2dad47f74e4dd, entries=150, sequenceid=232, filesize=11.9 K 2024-11-22T18:56:43,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/24263897fe244fd0b154156039cad35c as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/24263897fe244fd0b154156039cad35c 2024-11-22T18:56:43,249 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/24263897fe244fd0b154156039cad35c, entries=150, sequenceid=232, filesize=11.9 K 2024-11-22T18:56:43,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/c996fb88b9ec4f17bbc8ec9a70eb319a as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/c996fb88b9ec4f17bbc8ec9a70eb319a 2024-11-22T18:56:43,256 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/c996fb88b9ec4f17bbc8ec9a70eb319a, entries=150, sequenceid=232, filesize=11.9 K 2024-11-22T18:56:43,257 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 2f5686bd89d51ddd0842891a5a86fd98 in 1322ms, sequenceid=232, compaction requested=false 2024-11-22T18:56:43,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for 2f5686bd89d51ddd0842891a5a86fd98: 2024-11-22T18:56:43,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:43,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-22T18:56:43,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-22T18:56:43,261 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-22T18:56:43,261 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0900 sec 2024-11-22T18:56:43,263 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 2.0960 sec 2024-11-22T18:56:43,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-22T18:56:43,316 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T18:56:43,316 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T18:56:43,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-22T18:56:43,318 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T18:56:43,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-22T18:56:43,320 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T18:56:43,320 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T18:56:43,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-22T18:56:43,473 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-22T18:56:43,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:43,474 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing 2f5686bd89d51ddd0842891a5a86fd98 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-22T18:56:43,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/1a5b5b12ab4d44f9bdad3e8b60b58714 is 50, key is test_row_0/A:col10/1732301802860/Put/seqid=0 2024-11-22T18:56:43,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742087_1263 (size=12151) 2024-11-22T18:56:43,487 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/1a5b5b12ab4d44f9bdad3e8b60b58714 2024-11-22T18:56:43,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on 2f5686bd89d51ddd0842891a5a86fd98 2024-11-22T18:56:43,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. as already flushing 2024-11-22T18:56:43,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/73614efc75d14a58ade63244eadd11a0 is 50, key is test_row_0/B:col10/1732301802860/Put/seqid=0 2024-11-22T18:56:43,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742088_1264 (size=12151) 2024-11-22T18:56:43,507 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/73614efc75d14a58ade63244eadd11a0 2024-11-22T18:56:43,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/5e8a1b53c9c445cbb6b39aff26a5f239 is 50, key is test_row_0/C:col10/1732301802860/Put/seqid=0 2024-11-22T18:56:43,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742089_1265 (size=12151) 2024-11-22T18:56:43,521 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/5e8a1b53c9c445cbb6b39aff26a5f239 2024-11-22T18:56:43,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:43,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53306 deadline: 1732301863516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:43,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:43,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:43,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:43,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:43,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53250 deadline: 1732301863518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:43,523 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:43,523 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:43,523 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:43,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:43,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53340 deadline: 1732301863519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:43,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:43,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:43,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:43,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:43,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53324 deadline: 1732301863521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:43,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:43,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:43,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:43,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/1a5b5b12ab4d44f9bdad3e8b60b58714 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/1a5b5b12ab4d44f9bdad3e8b60b58714 2024-11-22T18:56:43,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:43,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53228 deadline: 1732301863522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:43,527 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:43,527 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:43,527 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:43,532 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/1a5b5b12ab4d44f9bdad3e8b60b58714, entries=150, sequenceid=249, filesize=11.9 K 2024-11-22T18:56:43,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/73614efc75d14a58ade63244eadd11a0 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/73614efc75d14a58ade63244eadd11a0 2024-11-22T18:56:43,537 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/73614efc75d14a58ade63244eadd11a0, entries=150, sequenceid=249, filesize=11.9 K 2024-11-22T18:56:43,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/5e8a1b53c9c445cbb6b39aff26a5f239 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/5e8a1b53c9c445cbb6b39aff26a5f239 2024-11-22T18:56:43,545 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/5e8a1b53c9c445cbb6b39aff26a5f239, entries=150, sequenceid=249, filesize=11.9 K 2024-11-22T18:56:43,546 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 2f5686bd89d51ddd0842891a5a86fd98 in 73ms, sequenceid=249, compaction requested=true 2024-11-22T18:56:43,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for 2f5686bd89d51ddd0842891a5a86fd98: 2024-11-22T18:56:43,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:43,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-22T18:56:43,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-22T18:56:43,550 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-22T18:56:43,550 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 227 msec 2024-11-22T18:56:43,552 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 234 msec 2024-11-22T18:56:43,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on 2f5686bd89d51ddd0842891a5a86fd98 2024-11-22T18:56:43,629 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T18:56:43,629 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2f5686bd89d51ddd0842891a5a86fd98 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-22T18:56:43,635 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/b6b7585abb594e11b658c8d248e7948f is 50, key is test_row_0/A:col10/1732301803511/Put/seqid=0 2024-11-22T18:56:43,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-22T18:56:43,636 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T18:56:43,637 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T18:56:43,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees 2024-11-22T18:56:43,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-22T18:56:43,638 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T18:56:43,639 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T18:56:43,639 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T18:56:43,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742090_1266 (size=14741) 2024-11-22T18:56:43,641 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/b6b7585abb594e11b658c8d248e7948f 2024-11-22T18:56:43,650 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/e50ddf21aead498285b2e8207715d0d3 is 50, key is test_row_0/B:col10/1732301803511/Put/seqid=0 2024-11-22T18:56:43,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:43,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53340 deadline: 1732301863645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:43,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:43,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:43,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:43,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53324 deadline: 1732301863645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:43,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:43,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:43,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:43,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:43,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:43,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53306 deadline: 1732301863645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:43,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:43,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:43,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:43,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:43,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53228 deadline: 1732301863645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:43,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:43,652 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:43,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:43,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:43,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53250 deadline: 1732301863646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:43,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:43,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:43,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:43,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742091_1267 (size=12301) 2024-11-22T18:56:43,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-22T18:56:43,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:43,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53340 deadline: 1732301863755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:43,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:43,757 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:43,757 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:43,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53324 deadline: 1732301863755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:43,757 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:43,758 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:43,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:43,758 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:43,758 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:43,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53306 deadline: 1732301863755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:43,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:43,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:43,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:43,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:43,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53228 deadline: 1732301863756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:43,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:43,758 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:43,758 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:43,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53250 deadline: 1732301863756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:43,758 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:43,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:43,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:43,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:43,792 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-22T18:56:43,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:43,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. as already flushing 2024-11-22T18:56:43,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:43,793 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:43,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:43,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:43,945 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-22T18:56:43,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:43,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. as already flushing 2024-11-22T18:56:43,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:43,946 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:43,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:43,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:43,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-22T18:56:43,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:43,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53340 deadline: 1732301863965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:43,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:43,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:43,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:43,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53324 deadline: 1732301863965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:43,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:43,968 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:43,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:43,968 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:43,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53306 deadline: 1732301863965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:43,968 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:43,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:43,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:43,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:43,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:43,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53228 deadline: 1732301863966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:43,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:43,968 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:43,968 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:43,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53250 deadline: 1732301863966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:43,968 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:43,969 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:43,969 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:43,969 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:44,056 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/e50ddf21aead498285b2e8207715d0d3 2024-11-22T18:56:44,064 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/145ffba30ecf4067a9f1450e83ff8ba9 is 50, key is test_row_0/C:col10/1732301803511/Put/seqid=0 2024-11-22T18:56:44,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742092_1268 (size=12301) 2024-11-22T18:56:44,099 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-22T18:56:44,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:44,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. as already flushing 2024-11-22T18:56:44,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:44,099 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:44,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:44,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:44,252 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-22T18:56:44,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:44,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. as already flushing 2024-11-22T18:56:44,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:44,252 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:44,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:44,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:44,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-22T18:56:44,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:44,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53340 deadline: 1732301864275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:44,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:44,280 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:44,280 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:44,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53306 deadline: 1732301864275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:44,280 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:44,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:44,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:44,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:44,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53324 deadline: 1732301864275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:44,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:44,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:44,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:44,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:44,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:44,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:44,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53228 deadline: 1732301864276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:44,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53250 deadline: 1732301864276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:44,281 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:44,281 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:44,281 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:44,281 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:44,281 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:44,281 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:44,405 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-22T18:56:44,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:44,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. as already flushing 2024-11-22T18:56:44,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:44,405 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:44,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:44,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:44,468 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/145ffba30ecf4067a9f1450e83ff8ba9 2024-11-22T18:56:44,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/b6b7585abb594e11b658c8d248e7948f as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/b6b7585abb594e11b658c8d248e7948f 2024-11-22T18:56:44,477 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/b6b7585abb594e11b658c8d248e7948f, entries=200, sequenceid=273, filesize=14.4 K 2024-11-22T18:56:44,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/e50ddf21aead498285b2e8207715d0d3 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/e50ddf21aead498285b2e8207715d0d3 2024-11-22T18:56:44,482 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/e50ddf21aead498285b2e8207715d0d3, entries=150, sequenceid=273, filesize=12.0 K 2024-11-22T18:56:44,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/145ffba30ecf4067a9f1450e83ff8ba9 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/145ffba30ecf4067a9f1450e83ff8ba9 2024-11-22T18:56:44,486 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/145ffba30ecf4067a9f1450e83ff8ba9, entries=150, sequenceid=273, filesize=12.0 K 2024-11-22T18:56:44,487 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 2f5686bd89d51ddd0842891a5a86fd98 in 858ms, sequenceid=273, compaction requested=true 2024-11-22T18:56:44,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2f5686bd89d51ddd0842891a5a86fd98: 2024-11-22T18:56:44,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2f5686bd89d51ddd0842891a5a86fd98:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T18:56:44,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:56:44,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2f5686bd89d51ddd0842891a5a86fd98:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T18:56:44,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:56:44,487 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T18:56:44,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2f5686bd89d51ddd0842891a5a86fd98:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T18:56:44,487 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T18:56:44,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:56:44,488 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49232 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T18:56:44,488 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51672 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T18:56:44,488 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1541): 2f5686bd89d51ddd0842891a5a86fd98/B is initiating minor compaction (all files) 2024-11-22T18:56:44,488 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1541): 2f5686bd89d51ddd0842891a5a86fd98/A is initiating minor compaction (all files) 2024-11-22T18:56:44,488 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2f5686bd89d51ddd0842891a5a86fd98/A in TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:44,488 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2f5686bd89d51ddd0842891a5a86fd98/B in TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:44,488 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/a7e10a7e98fb46c89694e1d36c900ed5, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/3d65d24a9e8b4e66a5b2dad47f74e4dd, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/1a5b5b12ab4d44f9bdad3e8b60b58714, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/b6b7585abb594e11b658c8d248e7948f] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp, totalSize=50.5 K 2024-11-22T18:56:44,489 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/e6af29262ff84900993340a58d392b49, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/24263897fe244fd0b154156039cad35c, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/73614efc75d14a58ade63244eadd11a0, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/e50ddf21aead498285b2e8207715d0d3] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp, totalSize=48.1 K 2024-11-22T18:56:44,489 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting a7e10a7e98fb46c89694e1d36c900ed5, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732301798371 2024-11-22T18:56:44,489 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting e6af29262ff84900993340a58d392b49, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732301798371 2024-11-22T18:56:44,489 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 24263897fe244fd0b154156039cad35c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1732301800688 2024-11-22T18:56:44,489 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d65d24a9e8b4e66a5b2dad47f74e4dd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1732301800688 2024-11-22T18:56:44,490 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a5b5b12ab4d44f9bdad3e8b60b58714, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1732301802858 2024-11-22T18:56:44,490 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6b7585abb594e11b658c8d248e7948f, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1732301803511 2024-11-22T18:56:44,490 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 73614efc75d14a58ade63244eadd11a0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1732301802858 2024-11-22T18:56:44,491 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting e50ddf21aead498285b2e8207715d0d3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1732301803511 2024-11-22T18:56:44,499 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2f5686bd89d51ddd0842891a5a86fd98#B#compaction#228 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:56:44,499 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/9803899540d44ca8892da796c6077051 is 50, key is test_row_0/B:col10/1732301803511/Put/seqid=0 2024-11-22T18:56:44,502 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2f5686bd89d51ddd0842891a5a86fd98#A#compaction#229 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:56:44,502 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/0632dde5db2c4d4d98c8e8bae04b6458 is 50, key is test_row_0/A:col10/1732301803511/Put/seqid=0 2024-11-22T18:56:44,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742093_1269 (size=12915) 2024-11-22T18:56:44,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742094_1270 (size=12915) 2024-11-22T18:56:44,515 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/0632dde5db2c4d4d98c8e8bae04b6458 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/0632dde5db2c4d4d98c8e8bae04b6458 2024-11-22T18:56:44,519 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2f5686bd89d51ddd0842891a5a86fd98/A of 2f5686bd89d51ddd0842891a5a86fd98 into 0632dde5db2c4d4d98c8e8bae04b6458(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:56:44,519 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2f5686bd89d51ddd0842891a5a86fd98: 2024-11-22T18:56:44,519 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., storeName=2f5686bd89d51ddd0842891a5a86fd98/A, priority=12, startTime=1732301804487; duration=0sec 2024-11-22T18:56:44,520 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:56:44,520 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2f5686bd89d51ddd0842891a5a86fd98:A 2024-11-22T18:56:44,520 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T18:56:44,521 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49232 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T18:56:44,521 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1541): 2f5686bd89d51ddd0842891a5a86fd98/C is initiating minor compaction (all files) 2024-11-22T18:56:44,521 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2f5686bd89d51ddd0842891a5a86fd98/C in TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:44,521 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/41dac32379f645b19829d3b92a4e960b, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/c996fb88b9ec4f17bbc8ec9a70eb319a, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/5e8a1b53c9c445cbb6b39aff26a5f239, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/145ffba30ecf4067a9f1450e83ff8ba9] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp, totalSize=48.1 K 2024-11-22T18:56:44,522 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41dac32379f645b19829d3b92a4e960b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732301798371 2024-11-22T18:56:44,522 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting c996fb88b9ec4f17bbc8ec9a70eb319a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1732301800688 2024-11-22T18:56:44,522 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e8a1b53c9c445cbb6b39aff26a5f239, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1732301802858 2024-11-22T18:56:44,523 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 145ffba30ecf4067a9f1450e83ff8ba9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1732301803511 2024-11-22T18:56:44,534 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2f5686bd89d51ddd0842891a5a86fd98#C#compaction#230 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:56:44,536 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/37c930a424f24de8b402750f71961501 is 50, key is test_row_0/C:col10/1732301803511/Put/seqid=0 2024-11-22T18:56:44,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742095_1271 (size=12915) 2024-11-22T18:56:44,544 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/37c930a424f24de8b402750f71961501 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/37c930a424f24de8b402750f71961501 2024-11-22T18:56:44,550 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2f5686bd89d51ddd0842891a5a86fd98/C of 2f5686bd89d51ddd0842891a5a86fd98 into 37c930a424f24de8b402750f71961501(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:56:44,551 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2f5686bd89d51ddd0842891a5a86fd98: 2024-11-22T18:56:44,551 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., storeName=2f5686bd89d51ddd0842891a5a86fd98/C, priority=12, startTime=1732301804487; duration=0sec 2024-11-22T18:56:44,551 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:56:44,551 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2f5686bd89d51ddd0842891a5a86fd98:C 2024-11-22T18:56:44,558 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-22T18:56:44,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:44,558 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2902): Flushing 2f5686bd89d51ddd0842891a5a86fd98 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-22T18:56:44,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/f02b337c54be4b54bfe4c0208fa4efdd is 50, key is test_row_0/A:col10/1732301803645/Put/seqid=0 2024-11-22T18:56:44,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742096_1272 (size=12301) 2024-11-22T18:56:44,570 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/f02b337c54be4b54bfe4c0208fa4efdd 2024-11-22T18:56:44,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/2734729b21c94a29b408f828e4eabe4a is 50, key is test_row_0/B:col10/1732301803645/Put/seqid=0 2024-11-22T18:56:44,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742097_1273 (size=12301) 2024-11-22T18:56:44,584 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/2734729b21c94a29b408f828e4eabe4a 2024-11-22T18:56:44,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/6491683126464d55b71b719290edbf63 is 50, key is test_row_0/C:col10/1732301803645/Put/seqid=0 2024-11-22T18:56:44,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742098_1274 (size=12301) 2024-11-22T18:56:44,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-22T18:56:44,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on 2f5686bd89d51ddd0842891a5a86fd98 2024-11-22T18:56:44,788 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. as already flushing 2024-11-22T18:56:44,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:44,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53306 deadline: 1732301864808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:44,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:44,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:44,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:44,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53340 deadline: 1732301864808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:44,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:44,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:44,814 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:44,814 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:44,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53250 deadline: 1732301864810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:44,814 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:44,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:44,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:44,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:44,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:44,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53228 deadline: 1732301864812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:44,816 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:44,816 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:44,816 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:44,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:44,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53324 deadline: 1732301864813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:44,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:44,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:44,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:44,910 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/9803899540d44ca8892da796c6077051 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/9803899540d44ca8892da796c6077051 2024-11-22T18:56:44,915 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2f5686bd89d51ddd0842891a5a86fd98/B of 2f5686bd89d51ddd0842891a5a86fd98 into 9803899540d44ca8892da796c6077051(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:56:44,915 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2f5686bd89d51ddd0842891a5a86fd98: 2024-11-22T18:56:44,915 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., storeName=2f5686bd89d51ddd0842891a5a86fd98/B, priority=12, startTime=1732301804487; duration=0sec 2024-11-22T18:56:44,915 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:56:44,916 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2f5686bd89d51ddd0842891a5a86fd98:B 2024-11-22T18:56:44,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:44,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53306 deadline: 1732301864915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:44,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:44,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53340 deadline: 1732301864915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:44,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:44,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:44,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:44,918 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:44,918 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:44,918 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:44,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:44,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53250 deadline: 1732301864925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:44,928 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:44,928 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:44,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:44,928 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:44,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53228 deadline: 1732301864926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:44,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:44,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:44,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:44,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:44,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53324 deadline: 1732301864926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:44,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:44,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:44,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:44,996 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/6491683126464d55b71b719290edbf63 2024-11-22T18:56:45,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/f02b337c54be4b54bfe4c0208fa4efdd as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/f02b337c54be4b54bfe4c0208fa4efdd 2024-11-22T18:56:45,005 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/f02b337c54be4b54bfe4c0208fa4efdd, entries=150, sequenceid=287, filesize=12.0 K 2024-11-22T18:56:45,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/2734729b21c94a29b408f828e4eabe4a as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/2734729b21c94a29b408f828e4eabe4a 2024-11-22T18:56:45,009 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/2734729b21c94a29b408f828e4eabe4a, entries=150, sequenceid=287, filesize=12.0 K 2024-11-22T18:56:45,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/6491683126464d55b71b719290edbf63 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/6491683126464d55b71b719290edbf63 2024-11-22T18:56:45,015 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/6491683126464d55b71b719290edbf63, entries=150, sequenceid=287, filesize=12.0 K 2024-11-22T18:56:45,016 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 2f5686bd89d51ddd0842891a5a86fd98 in 457ms, sequenceid=287, compaction requested=false 2024-11-22T18:56:45,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2603): Flush status journal for 2f5686bd89d51ddd0842891a5a86fd98: 2024-11-22T18:56:45,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:45,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=85 2024-11-22T18:56:45,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster(4169): Remote procedure done, pid=85 2024-11-22T18:56:45,019 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-22T18:56:45,019 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3780 sec 2024-11-22T18:56:45,021 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees in 1.3830 sec 2024-11-22T18:56:45,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on 2f5686bd89d51ddd0842891a5a86fd98 2024-11-22T18:56:45,128 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T18:56:45,128 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2f5686bd89d51ddd0842891a5a86fd98 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-22T18:56:45,134 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/f9f428bfdd0940d5b574872f78b4a42f is 50, key is test_row_0/A:col10/1732301805127/Put/seqid=0 2024-11-22T18:56:45,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742099_1275 (size=14741) 2024-11-22T18:56:45,140 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/f9f428bfdd0940d5b574872f78b4a42f 2024-11-22T18:56:45,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:45,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53306 deadline: 1732301865137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:45,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:45,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:45,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:45,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:45,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53340 deadline: 1732301865138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:45,143 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:45,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:45,144 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:45,144 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:45,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53250 deadline: 1732301865139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:45,144 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:45,144 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:45,144 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:45,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:45,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53324 deadline: 1732301865141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:45,147 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:45,147 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:45,147 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:45,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:45,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53228 deadline: 1732301865141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:45,147 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:45,147 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:45,147 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:45,150 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/75b0c112fc0b4b45a357fa90fde33e03 is 50, key is test_row_0/B:col10/1732301805127/Put/seqid=0 2024-11-22T18:56:45,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742100_1276 (size=12301) 2024-11-22T18:56:45,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:45,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53306 deadline: 1732301865246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:45,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:45,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53340 deadline: 1732301865246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:45,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:45,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:45,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:45,249 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:45,249 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:45,249 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:45,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:45,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53250 deadline: 1732301865256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:45,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:45,260 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:45,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:45,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53324 deadline: 1732301865256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:45,260 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:45,260 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:45,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53228 deadline: 1732301865256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:45,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:45,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:45,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:45,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:45,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:45,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:45,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:45,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53306 deadline: 1732301865456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:45,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:45,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:45,459 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:45,459 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:45,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53340 deadline: 1732301865456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:45,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:45,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:45,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:45,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:45,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53250 deadline: 1732301865466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:45,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:45,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:45,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:45,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53324 deadline: 1732301865466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:45,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:45,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:45,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:45,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53228 deadline: 1732301865466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:45,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:45,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:45,470 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:45,470 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:45,470 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:45,556 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/75b0c112fc0b4b45a357fa90fde33e03 2024-11-22T18:56:45,564 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/654d713dce88401a9d63cd178464f81b is 50, key is test_row_0/C:col10/1732301805127/Put/seqid=0 2024-11-22T18:56:45,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742101_1277 (size=12301) 2024-11-22T18:56:45,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:45,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53306 deadline: 1732301865765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:45,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:45,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53340 deadline: 1732301865766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:45,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:45,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:45,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:45,768 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:45,768 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:45,768 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:45,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:45,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53250 deadline: 1732301865776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:45,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:45,779 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:45,779 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:45,779 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:45,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53324 deadline: 1732301865776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:45,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:45,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:45,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:45,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:45,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53228 deadline: 1732301865776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:45,780 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:45,780 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:45,780 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:45,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-22T18:56:45,785 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T18:56:45,786 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T18:56:45,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees 2024-11-22T18:56:45,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-22T18:56:45,788 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T18:56:45,789 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T18:56:45,790 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T18:56:45,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-22T18:56:45,942 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-22T18:56:45,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:45,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. as already flushing 2024-11-22T18:56:45,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:45,942 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:45,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:45,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:45,971 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/654d713dce88401a9d63cd178464f81b 2024-11-22T18:56:45,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/f9f428bfdd0940d5b574872f78b4a42f as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/f9f428bfdd0940d5b574872f78b4a42f 2024-11-22T18:56:45,980 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/f9f428bfdd0940d5b574872f78b4a42f, entries=200, sequenceid=314, filesize=14.4 K 2024-11-22T18:56:45,980 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/75b0c112fc0b4b45a357fa90fde33e03 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/75b0c112fc0b4b45a357fa90fde33e03 2024-11-22T18:56:45,984 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/75b0c112fc0b4b45a357fa90fde33e03, entries=150, sequenceid=314, filesize=12.0 K 2024-11-22T18:56:45,985 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/654d713dce88401a9d63cd178464f81b as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/654d713dce88401a9d63cd178464f81b 2024-11-22T18:56:45,989 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/654d713dce88401a9d63cd178464f81b, entries=150, sequenceid=314, filesize=12.0 K 2024-11-22T18:56:45,990 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 2f5686bd89d51ddd0842891a5a86fd98 in 862ms, sequenceid=314, compaction requested=true 2024-11-22T18:56:45,990 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2f5686bd89d51ddd0842891a5a86fd98: 2024-11-22T18:56:45,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2f5686bd89d51ddd0842891a5a86fd98:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T18:56:45,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:56:45,990 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T18:56:45,990 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T18:56:45,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2f5686bd89d51ddd0842891a5a86fd98:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T18:56:45,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:56:45,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2f5686bd89d51ddd0842891a5a86fd98:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T18:56:45,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:56:45,991 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T18:56:45,992 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1541): 2f5686bd89d51ddd0842891a5a86fd98/B is initiating minor compaction (all files) 2024-11-22T18:56:45,992 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2f5686bd89d51ddd0842891a5a86fd98/B in TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:45,992 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/9803899540d44ca8892da796c6077051, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/2734729b21c94a29b408f828e4eabe4a, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/75b0c112fc0b4b45a357fa90fde33e03] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp, totalSize=36.6 K 2024-11-22T18:56:45,992 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39957 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T18:56:45,992 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1541): 2f5686bd89d51ddd0842891a5a86fd98/A is initiating minor compaction (all files) 2024-11-22T18:56:45,992 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2f5686bd89d51ddd0842891a5a86fd98/A in TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:45,992 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/0632dde5db2c4d4d98c8e8bae04b6458, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/f02b337c54be4b54bfe4c0208fa4efdd, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/f9f428bfdd0940d5b574872f78b4a42f] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp, totalSize=39.0 K 2024-11-22T18:56:45,992 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 9803899540d44ca8892da796c6077051, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1732301803511 2024-11-22T18:56:45,993 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0632dde5db2c4d4d98c8e8bae04b6458, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1732301803511 2024-11-22T18:56:45,993 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 2734729b21c94a29b408f828e4eabe4a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732301803639 2024-11-22T18:56:45,993 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 75b0c112fc0b4b45a357fa90fde33e03, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1732301804811 2024-11-22T18:56:45,993 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting f02b337c54be4b54bfe4c0208fa4efdd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732301803639 2024-11-22T18:56:45,994 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9f428bfdd0940d5b574872f78b4a42f, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1732301804810 2024-11-22T18:56:46,003 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2f5686bd89d51ddd0842891a5a86fd98#A#compaction#237 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:56:46,003 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/36d452d0b4fe4f8297b9af6297a7ed50 is 50, key is test_row_0/A:col10/1732301805127/Put/seqid=0 2024-11-22T18:56:46,006 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2f5686bd89d51ddd0842891a5a86fd98#B#compaction#238 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:56:46,007 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/20d88c7e0e634381b08c3ac56b871044 is 50, key is test_row_0/B:col10/1732301805127/Put/seqid=0 2024-11-22T18:56:46,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742102_1278 (size=13017) 2024-11-22T18:56:46,017 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/36d452d0b4fe4f8297b9af6297a7ed50 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/36d452d0b4fe4f8297b9af6297a7ed50 2024-11-22T18:56:46,022 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2f5686bd89d51ddd0842891a5a86fd98/A of 2f5686bd89d51ddd0842891a5a86fd98 into 36d452d0b4fe4f8297b9af6297a7ed50(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:56:46,022 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2f5686bd89d51ddd0842891a5a86fd98: 2024-11-22T18:56:46,022 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., storeName=2f5686bd89d51ddd0842891a5a86fd98/A, priority=13, startTime=1732301805990; duration=0sec 2024-11-22T18:56:46,022 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:56:46,022 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2f5686bd89d51ddd0842891a5a86fd98:A 2024-11-22T18:56:46,022 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T18:56:46,023 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T18:56:46,023 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1541): 2f5686bd89d51ddd0842891a5a86fd98/C is initiating minor compaction (all files) 2024-11-22T18:56:46,023 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2f5686bd89d51ddd0842891a5a86fd98/C in TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:46,024 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/37c930a424f24de8b402750f71961501, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/6491683126464d55b71b719290edbf63, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/654d713dce88401a9d63cd178464f81b] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp, totalSize=36.6 K 2024-11-22T18:56:46,024 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37c930a424f24de8b402750f71961501, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1732301803511 2024-11-22T18:56:46,024 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6491683126464d55b71b719290edbf63, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732301803639 2024-11-22T18:56:46,025 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 654d713dce88401a9d63cd178464f81b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1732301804811 2024-11-22T18:56:46,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742103_1279 (size=13017) 2024-11-22T18:56:46,032 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/20d88c7e0e634381b08c3ac56b871044 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/20d88c7e0e634381b08c3ac56b871044 2024-11-22T18:56:46,039 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2f5686bd89d51ddd0842891a5a86fd98/B of 2f5686bd89d51ddd0842891a5a86fd98 into 20d88c7e0e634381b08c3ac56b871044(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:56:46,039 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2f5686bd89d51ddd0842891a5a86fd98: 2024-11-22T18:56:46,039 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., storeName=2f5686bd89d51ddd0842891a5a86fd98/B, priority=13, startTime=1732301805990; duration=0sec 2024-11-22T18:56:46,039 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:56:46,039 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2f5686bd89d51ddd0842891a5a86fd98:B 2024-11-22T18:56:46,046 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2f5686bd89d51ddd0842891a5a86fd98#C#compaction#239 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:56:46,047 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/cafa9dad25c741189b2352a7b2073bfe is 50, key is test_row_0/C:col10/1732301805127/Put/seqid=0 2024-11-22T18:56:46,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742104_1280 (size=13017) 2024-11-22T18:56:46,059 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/cafa9dad25c741189b2352a7b2073bfe as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/cafa9dad25c741189b2352a7b2073bfe 2024-11-22T18:56:46,066 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2f5686bd89d51ddd0842891a5a86fd98/C of 2f5686bd89d51ddd0842891a5a86fd98 into cafa9dad25c741189b2352a7b2073bfe(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:56:46,066 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2f5686bd89d51ddd0842891a5a86fd98: 2024-11-22T18:56:46,066 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., storeName=2f5686bd89d51ddd0842891a5a86fd98/C, priority=13, startTime=1732301805991; duration=0sec 2024-11-22T18:56:46,067 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:56:46,067 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2f5686bd89d51ddd0842891a5a86fd98:C 2024-11-22T18:56:46,095 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-22T18:56:46,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:46,096 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2902): Flushing 2f5686bd89d51ddd0842891a5a86fd98 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-22T18:56:46,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/ce57e715135e4f0ebe94536d83c4b370 is 50, key is test_row_0/A:col10/1732301805139/Put/seqid=0 2024-11-22T18:56:46,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-22T18:56:46,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742105_1281 (size=12301) 2024-11-22T18:56:46,106 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/ce57e715135e4f0ebe94536d83c4b370 2024-11-22T18:56:46,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/b7a34ecc207047d3875cb5a2d41c316c is 50, key is test_row_0/B:col10/1732301805139/Put/seqid=0 2024-11-22T18:56:46,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742106_1282 (size=12301) 2024-11-22T18:56:46,151 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/b7a34ecc207047d3875cb5a2d41c316c 2024-11-22T18:56:46,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/b6360ca7cf334a49bc3f3d368deb53ce is 50, key is test_row_0/C:col10/1732301805139/Put/seqid=0 2024-11-22T18:56:46,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742107_1283 (size=12301) 2024-11-22T18:56:46,174 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/b6360ca7cf334a49bc3f3d368deb53ce 2024-11-22T18:56:46,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/ce57e715135e4f0ebe94536d83c4b370 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/ce57e715135e4f0ebe94536d83c4b370 2024-11-22T18:56:46,185 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/ce57e715135e4f0ebe94536d83c4b370, entries=150, sequenceid=327, filesize=12.0 K 2024-11-22T18:56:46,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/b7a34ecc207047d3875cb5a2d41c316c as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/b7a34ecc207047d3875cb5a2d41c316c 2024-11-22T18:56:46,190 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/b7a34ecc207047d3875cb5a2d41c316c, entries=150, sequenceid=327, filesize=12.0 K 2024-11-22T18:56:46,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/b6360ca7cf334a49bc3f3d368deb53ce as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/b6360ca7cf334a49bc3f3d368deb53ce 2024-11-22T18:56:46,195 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/b6360ca7cf334a49bc3f3d368deb53ce, entries=150, sequenceid=327, filesize=12.0 K 2024-11-22T18:56:46,196 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=0 B/0 for 2f5686bd89d51ddd0842891a5a86fd98 in 100ms, sequenceid=327, compaction requested=false 2024-11-22T18:56:46,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2603): Flush status journal for 2f5686bd89d51ddd0842891a5a86fd98: 2024-11-22T18:56:46,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:46,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=87 2024-11-22T18:56:46,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster(4169): Remote procedure done, pid=87 2024-11-22T18:56:46,200 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-22T18:56:46,200 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 408 msec 2024-11-22T18:56:46,202 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees in 414 msec 2024-11-22T18:56:46,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on 2f5686bd89d51ddd0842891a5a86fd98 2024-11-22T18:56:46,287 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T18:56:46,287 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2f5686bd89d51ddd0842891a5a86fd98 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-22T18:56:46,294 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/22e0685c9db04905a0d5996eb43ffa6b is 50, key is test_row_0/A:col10/1732301806284/Put/seqid=0 2024-11-22T18:56:46,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742108_1284 (size=12301) 2024-11-22T18:56:46,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:46,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53250 deadline: 1732301866313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:46,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:46,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:46,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:46,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53324 deadline: 1732301866314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:46,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:46,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:46,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53340 deadline: 1732301866314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:46,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:46,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:46,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:46,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:46,320 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:46,320 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:46,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53228 deadline: 1732301866315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:46,320 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:46,320 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:46,320 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:46,320 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:46,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:46,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53306 deadline: 1732301866316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:46,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:46,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:46,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:46,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-22T18:56:46,415 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T18:56:46,416 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T18:56:46,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees 2024-11-22T18:56:46,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-22T18:56:46,418 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=88, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T18:56:46,418 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=88, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T18:56:46,419 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T18:56:46,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:46,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53250 deadline: 1732301866425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:46,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:46,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53324 deadline: 1732301866425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:46,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:46,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:46,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:46,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:46,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:46,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:46,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53340 deadline: 1732301866425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:46,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:46,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:46,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53228 deadline: 1732301866425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:46,430 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:46,430 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:46,430 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:46,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:46,430 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:46,430 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:46,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53306 deadline: 1732301866425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:46,430 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:46,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:46,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:46,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:46,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-22T18:56:46,571 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-22T18:56:46,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:46,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. as already flushing 2024-11-22T18:56:46,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:46,571 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:46,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:46,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:46,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:46,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53250 deadline: 1732301866635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:46,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:46,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:46,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53324 deadline: 1732301866635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:46,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:46,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:46,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:46,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:46,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:46,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53340 deadline: 1732301866636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:46,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:46,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:46,639 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:46,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53228 deadline: 1732301866636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:46,639 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:46,639 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:46,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:46,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:46,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:46,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53306 deadline: 1732301866636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:46,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:46,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:46,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:46,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:46,702 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/22e0685c9db04905a0d5996eb43ffa6b 2024-11-22T18:56:46,709 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/473a825efbcb4b33862a66024acb525f is 50, key is test_row_0/B:col10/1732301806284/Put/seqid=0 2024-11-22T18:56:46,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742109_1285 (size=12301) 2024-11-22T18:56:46,714 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/473a825efbcb4b33862a66024acb525f 2024-11-22T18:56:46,722 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/e539b75cda9649dc998259c12bc992c5 is 50, key is test_row_0/C:col10/1732301806284/Put/seqid=0 2024-11-22T18:56:46,724 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-22T18:56:46,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:46,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. as already flushing 2024-11-22T18:56:46,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:46,724 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:46,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:46,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742110_1286 (size=12301) 2024-11-22T18:56:46,726 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/e539b75cda9649dc998259c12bc992c5 2024-11-22T18:56:46,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:46,731 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/22e0685c9db04905a0d5996eb43ffa6b as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/22e0685c9db04905a0d5996eb43ffa6b 2024-11-22T18:56:46,736 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/22e0685c9db04905a0d5996eb43ffa6b, entries=150, sequenceid=338, filesize=12.0 K 2024-11-22T18:56:46,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-22T18:56:46,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/473a825efbcb4b33862a66024acb525f as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/473a825efbcb4b33862a66024acb525f 2024-11-22T18:56:46,741 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/473a825efbcb4b33862a66024acb525f, entries=150, sequenceid=338, filesize=12.0 K 2024-11-22T18:56:46,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/e539b75cda9649dc998259c12bc992c5 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/e539b75cda9649dc998259c12bc992c5 2024-11-22T18:56:46,746 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/e539b75cda9649dc998259c12bc992c5, entries=150, sequenceid=338, filesize=12.0 K 2024-11-22T18:56:46,747 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 2f5686bd89d51ddd0842891a5a86fd98 in 460ms, sequenceid=338, compaction requested=true 2024-11-22T18:56:46,747 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2f5686bd89d51ddd0842891a5a86fd98: 2024-11-22T18:56:46,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2f5686bd89d51ddd0842891a5a86fd98:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T18:56:46,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:56:46,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2f5686bd89d51ddd0842891a5a86fd98:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T18:56:46,748 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T18:56:46,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:56:46,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2f5686bd89d51ddd0842891a5a86fd98:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T18:56:46,748 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T18:56:46,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:56:46,749 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T18:56:46,749 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T18:56:46,749 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1541): 2f5686bd89d51ddd0842891a5a86fd98/A is initiating minor compaction (all files) 2024-11-22T18:56:46,749 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1541): 2f5686bd89d51ddd0842891a5a86fd98/B is initiating minor compaction (all files) 2024-11-22T18:56:46,749 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2f5686bd89d51ddd0842891a5a86fd98/A in TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:46,749 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2f5686bd89d51ddd0842891a5a86fd98/B in TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:46,749 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/36d452d0b4fe4f8297b9af6297a7ed50, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/ce57e715135e4f0ebe94536d83c4b370, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/22e0685c9db04905a0d5996eb43ffa6b] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp, totalSize=36.7 K 2024-11-22T18:56:46,749 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/20d88c7e0e634381b08c3ac56b871044, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/b7a34ecc207047d3875cb5a2d41c316c, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/473a825efbcb4b33862a66024acb525f] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp, totalSize=36.7 K 2024-11-22T18:56:46,749 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 20d88c7e0e634381b08c3ac56b871044, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1732301804811 2024-11-22T18:56:46,749 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36d452d0b4fe4f8297b9af6297a7ed50, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1732301804811 2024-11-22T18:56:46,749 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting b7a34ecc207047d3875cb5a2d41c316c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1732301805130 2024-11-22T18:56:46,750 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce57e715135e4f0ebe94536d83c4b370, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1732301805130 2024-11-22T18:56:46,750 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 473a825efbcb4b33862a66024acb525f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1732301806279 2024-11-22T18:56:46,751 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22e0685c9db04905a0d5996eb43ffa6b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1732301806279 2024-11-22T18:56:46,759 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2f5686bd89d51ddd0842891a5a86fd98#B#compaction#246 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:56:46,760 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/cf16dd29fa144f15ba7ba6c1a469c743 is 50, key is test_row_0/B:col10/1732301806284/Put/seqid=0 2024-11-22T18:56:46,762 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2f5686bd89d51ddd0842891a5a86fd98#A#compaction#247 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:56:46,763 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/f2ac0dfde571496db79625bf6ef4a82a is 50, key is test_row_0/A:col10/1732301806284/Put/seqid=0 2024-11-22T18:56:46,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742111_1287 (size=13119) 2024-11-22T18:56:46,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742112_1288 (size=13119) 2024-11-22T18:56:46,877 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-22T18:56:46,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:46,878 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2902): Flushing 2f5686bd89d51ddd0842891a5a86fd98 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-22T18:56:46,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/9cefde34d7b64fdcbb76fe085ca09e46 is 50, key is test_row_0/A:col10/1732301806314/Put/seqid=0 2024-11-22T18:56:46,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742113_1289 (size=12301) 2024-11-22T18:56:46,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on 2f5686bd89d51ddd0842891a5a86fd98 2024-11-22T18:56:46,948 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. as already flushing 2024-11-22T18:56:46,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:46,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53306 deadline: 1732301866951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:46,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:46,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:46,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:46,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:46,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53250 deadline: 1732301866954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:46,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:46,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:46,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:46,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:46,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53340 deadline: 1732301866956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:46,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:46,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53324 deadline: 1732301866957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:46,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:46,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:46,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:46,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:46,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53228 deadline: 1732301866957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:46,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:46,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:46,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:46,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:46,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:46,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:47,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-22T18:56:47,068 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:47,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53306 deadline: 1732301867066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:47,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:47,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:47,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:47,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:47,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53250 deadline: 1732301867066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:47,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:47,069 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:47,069 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:47,069 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:47,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53340 deadline: 1732301867066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:47,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:47,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53228 deadline: 1732301867066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:47,070 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:47,070 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:47,070 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:47,070 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:47,070 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:47,070 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:47,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:47,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53324 deadline: 1732301867067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:47,070 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:47,070 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:47,070 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:47,181 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/f2ac0dfde571496db79625bf6ef4a82a as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/f2ac0dfde571496db79625bf6ef4a82a 2024-11-22T18:56:47,182 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/cf16dd29fa144f15ba7ba6c1a469c743 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/cf16dd29fa144f15ba7ba6c1a469c743 2024-11-22T18:56:47,187 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2f5686bd89d51ddd0842891a5a86fd98/A of 2f5686bd89d51ddd0842891a5a86fd98 into f2ac0dfde571496db79625bf6ef4a82a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:56:47,187 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2f5686bd89d51ddd0842891a5a86fd98: 2024-11-22T18:56:47,187 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., storeName=2f5686bd89d51ddd0842891a5a86fd98/A, priority=13, startTime=1732301806747; duration=0sec 2024-11-22T18:56:47,187 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:56:47,187 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2f5686bd89d51ddd0842891a5a86fd98:A 2024-11-22T18:56:47,187 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T18:56:47,187 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2f5686bd89d51ddd0842891a5a86fd98/B of 2f5686bd89d51ddd0842891a5a86fd98 into cf16dd29fa144f15ba7ba6c1a469c743(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:56:47,187 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2f5686bd89d51ddd0842891a5a86fd98: 2024-11-22T18:56:47,188 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., storeName=2f5686bd89d51ddd0842891a5a86fd98/B, priority=13, startTime=1732301806748; duration=0sec 2024-11-22T18:56:47,188 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:56:47,188 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2f5686bd89d51ddd0842891a5a86fd98:B 2024-11-22T18:56:47,188 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T18:56:47,189 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1541): 2f5686bd89d51ddd0842891a5a86fd98/C is initiating minor compaction (all files) 2024-11-22T18:56:47,189 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2f5686bd89d51ddd0842891a5a86fd98/C in TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:47,189 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/cafa9dad25c741189b2352a7b2073bfe, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/b6360ca7cf334a49bc3f3d368deb53ce, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/e539b75cda9649dc998259c12bc992c5] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp, totalSize=36.7 K 2024-11-22T18:56:47,189 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting cafa9dad25c741189b2352a7b2073bfe, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1732301804811 2024-11-22T18:56:47,189 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6360ca7cf334a49bc3f3d368deb53ce, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1732301805130 2024-11-22T18:56:47,190 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting e539b75cda9649dc998259c12bc992c5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1732301806279 2024-11-22T18:56:47,196 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2f5686bd89d51ddd0842891a5a86fd98#C#compaction#249 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:56:47,196 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/4037cf78c7d84aafa28d43acbd987477 is 50, key is test_row_0/C:col10/1732301806284/Put/seqid=0 2024-11-22T18:56:47,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742114_1290 (size=13119) 2024-11-22T18:56:47,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:47,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53340 deadline: 1732301867277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:47,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:47,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53250 deadline: 1732301867277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:47,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:47,280 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:47,280 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:47,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53228 deadline: 1732301867277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:47,280 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:47,280 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:47,280 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:47,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:47,280 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:47,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53306 deadline: 1732301867277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:47,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:47,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53324 deadline: 1732301867277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:47,280 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:47,280 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:47,280 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:47,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:47,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:47,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:47,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:47,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:47,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:47,288 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/9cefde34d7b64fdcbb76fe085ca09e46 2024-11-22T18:56:47,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/f751926882f54cca865e6e29935cbfe7 is 50, key is test_row_0/B:col10/1732301806314/Put/seqid=0 2024-11-22T18:56:47,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742115_1291 (size=12301) 2024-11-22T18:56:47,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-22T18:56:47,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:47,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53340 deadline: 1732301867585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:47,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:47,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:47,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:47,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53250 deadline: 1732301867585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:47,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:47,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:47,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:47,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:47,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:47,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53228 deadline: 1732301867585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:47,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:47,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:47,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:47,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53306 deadline: 1732301867585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:47,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:47,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:47,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:47,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:47,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53324 deadline: 1732301867585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:47,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:47,588 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:47,588 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:47,588 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:47,606 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/4037cf78c7d84aafa28d43acbd987477 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/4037cf78c7d84aafa28d43acbd987477 2024-11-22T18:56:47,618 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2f5686bd89d51ddd0842891a5a86fd98/C of 2f5686bd89d51ddd0842891a5a86fd98 into 4037cf78c7d84aafa28d43acbd987477(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:56:47,618 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2f5686bd89d51ddd0842891a5a86fd98: 2024-11-22T18:56:47,618 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., storeName=2f5686bd89d51ddd0842891a5a86fd98/C, priority=13, startTime=1732301806748; duration=0sec 2024-11-22T18:56:47,618 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:56:47,618 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2f5686bd89d51ddd0842891a5a86fd98:C 2024-11-22T18:56:47,701 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/f751926882f54cca865e6e29935cbfe7 2024-11-22T18:56:47,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/f9af0f753a26490c93810a4e78ea63dd is 50, key is test_row_0/C:col10/1732301806314/Put/seqid=0 2024-11-22T18:56:47,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742116_1292 (size=12301) 2024-11-22T18:56:48,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:48,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53340 deadline: 1732301868095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:48,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:48,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53250 deadline: 1732301868095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:48,097 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:48,097 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:48,097 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:48,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:48,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:48,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53228 deadline: 1732301868095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:48,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:48,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:48,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:48,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53306 deadline: 1732301868096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:48,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:48,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:48,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:48,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:48,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53324 deadline: 1732301868096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:48,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:48,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:48,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:48,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:48,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:48,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:48,114 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/f9af0f753a26490c93810a4e78ea63dd 2024-11-22T18:56:48,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/9cefde34d7b64fdcbb76fe085ca09e46 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/9cefde34d7b64fdcbb76fe085ca09e46 2024-11-22T18:56:48,122 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/9cefde34d7b64fdcbb76fe085ca09e46, entries=150, sequenceid=363, filesize=12.0 K 2024-11-22T18:56:48,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/f751926882f54cca865e6e29935cbfe7 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/f751926882f54cca865e6e29935cbfe7 2024-11-22T18:56:48,128 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/f751926882f54cca865e6e29935cbfe7, entries=150, sequenceid=363, filesize=12.0 K 2024-11-22T18:56:48,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/f9af0f753a26490c93810a4e78ea63dd as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/f9af0f753a26490c93810a4e78ea63dd 2024-11-22T18:56:48,133 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/f9af0f753a26490c93810a4e78ea63dd, entries=150, sequenceid=363, filesize=12.0 K 2024-11-22T18:56:48,133 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 2f5686bd89d51ddd0842891a5a86fd98 in 1255ms, sequenceid=363, compaction requested=false 2024-11-22T18:56:48,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2603): Flush status journal for 2f5686bd89d51ddd0842891a5a86fd98: 2024-11-22T18:56:48,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:48,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=89 2024-11-22T18:56:48,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster(4169): Remote procedure done, pid=89 2024-11-22T18:56:48,137 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-22T18:56:48,137 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7170 sec 2024-11-22T18:56:48,139 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees in 1.7220 sec 2024-11-22T18:56:48,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-22T18:56:48,576 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T18:56:48,576 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T18:56:48,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees 2024-11-22T18:56:48,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-22T18:56:48,578 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=90, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T18:56:48,579 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=90, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T18:56:48,579 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T18:56:48,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-22T18:56:48,732 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-22T18:56:48,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:48,733 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2902): Flushing 2f5686bd89d51ddd0842891a5a86fd98 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-22T18:56:48,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/776bb2ac6ca747438b299ec15837dccd is 50, key is test_row_0/A:col10/1732301806949/Put/seqid=0 2024-11-22T18:56:48,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742117_1293 (size=12301) 2024-11-22T18:56:48,746 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/776bb2ac6ca747438b299ec15837dccd 2024-11-22T18:56:48,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/688a94ac74cf4525919648560fd0935c is 50, key is test_row_0/B:col10/1732301806949/Put/seqid=0 2024-11-22T18:56:48,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742118_1294 (size=12301) 2024-11-22T18:56:48,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-22T18:56:49,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on 2f5686bd89d51ddd0842891a5a86fd98 2024-11-22T18:56:49,108 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. as already flushing 2024-11-22T18:56:49,146 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:49,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53250 deadline: 1732301869140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:49,146 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:49,146 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:49,146 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:49,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:49,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53306 deadline: 1732301869141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:49,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:49,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:49,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:49,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53228 deadline: 1732301869143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:49,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:49,147 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:49,147 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:49,147 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:49,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:49,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53340 deadline: 1732301869146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:49,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:49,151 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:49,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:49,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53324 deadline: 1732301869146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:49,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:49,152 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:49,152 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:49,152 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:49,166 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/688a94ac74cf4525919648560fd0935c 2024-11-22T18:56:49,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/54713e67cdd241eb9632e0b4fe9ba71b is 50, key is test_row_0/C:col10/1732301806949/Put/seqid=0 2024-11-22T18:56:49,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742119_1295 (size=12301) 2024-11-22T18:56:49,204 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/54713e67cdd241eb9632e0b4fe9ba71b 2024-11-22T18:56:49,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-22T18:56:49,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/776bb2ac6ca747438b299ec15837dccd as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/776bb2ac6ca747438b299ec15837dccd 2024-11-22T18:56:49,218 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/776bb2ac6ca747438b299ec15837dccd, entries=150, sequenceid=378, filesize=12.0 K 2024-11-22T18:56:49,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/688a94ac74cf4525919648560fd0935c as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/688a94ac74cf4525919648560fd0935c 2024-11-22T18:56:49,235 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/688a94ac74cf4525919648560fd0935c, entries=150, sequenceid=378, filesize=12.0 K 2024-11-22T18:56:49,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/54713e67cdd241eb9632e0b4fe9ba71b as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/54713e67cdd241eb9632e0b4fe9ba71b 2024-11-22T18:56:49,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:49,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53228 deadline: 1732301869256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:49,258 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:49,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:49,258 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:49,259 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:49,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53250 deadline: 1732301869256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:49,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:49,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53324 deadline: 1732301869256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:49,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:49,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:49,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:49,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:49,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53306 deadline: 1732301869257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:49,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:49,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:49,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:49,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:49,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53340 deadline: 1732301869257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:49,260 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:49,260 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:49,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:49,260 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:49,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:49,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:49,267 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/54713e67cdd241eb9632e0b4fe9ba71b, entries=150, sequenceid=378, filesize=12.0 K 2024-11-22T18:56:49,268 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 2f5686bd89d51ddd0842891a5a86fd98 in 536ms, sequenceid=378, compaction requested=true 2024-11-22T18:56:49,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2603): Flush status journal for 2f5686bd89d51ddd0842891a5a86fd98: 2024-11-22T18:56:49,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:49,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=91 2024-11-22T18:56:49,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster(4169): Remote procedure done, pid=91 2024-11-22T18:56:49,272 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-22T18:56:49,272 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 691 msec 2024-11-22T18:56:49,275 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees in 697 msec 2024-11-22T18:56:49,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on 2f5686bd89d51ddd0842891a5a86fd98 2024-11-22T18:56:49,469 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T18:56:49,469 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2f5686bd89d51ddd0842891a5a86fd98 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-22T18:56:49,475 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/d254867f8cde4f58ab72c0a1816596be is 50, key is test_row_0/A:col10/1732301809144/Put/seqid=0 2024-11-22T18:56:49,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742120_1296 (size=12301) 2024-11-22T18:56:49,480 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/d254867f8cde4f58ab72c0a1816596be 2024-11-22T18:56:49,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:49,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53250 deadline: 1732301869476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:49,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:49,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53228 deadline: 1732301869476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:49,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:49,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53324 deadline: 1732301869478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:49,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:49,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53306 deadline: 1732301869480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:49,485 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:49,485 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:49,485 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:49,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:49,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:49,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53340 deadline: 1732301869480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:49,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:49,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:49,485 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:49,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:49,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:49,485 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:49,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:49,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:49,485 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:49,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:49,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:49,489 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/0d59d00df4fe4045b03595bebf119f3d is 50, key is test_row_0/B:col10/1732301809144/Put/seqid=0 2024-11-22T18:56:49,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742121_1297 (size=12301) 2024-11-22T18:56:49,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:49,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53228 deadline: 1732301869595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:49,598 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:49,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:49,598 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:49,598 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:49,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53250 deadline: 1732301869595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:49,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:49,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:49,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53324 deadline: 1732301869596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:49,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:49,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:49,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:49,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:49,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:49,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:49,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53340 deadline: 1732301869596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:49,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:49,598 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:49,598 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:49,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53306 deadline: 1732301869596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:49,598 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:49,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:49,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:49,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:49,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-22T18:56:49,716 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T18:56:49,716 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T18:56:49,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] procedure2.ProcedureExecutor(1139): Stored pid=92, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=92, table=TestAcidGuarantees 2024-11-22T18:56:49,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-22T18:56:49,718 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=92, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=92, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T18:56:49,719 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=92, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=92, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T18:56:49,719 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T18:56:49,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:49,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53228 deadline: 1732301869806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:49,809 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:49,809 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:49,809 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:49,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:49,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53250 deadline: 1732301869806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:49,809 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:49,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:49,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:49,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:49,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53324 deadline: 1732301869806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:49,810 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:49,810 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:49,810 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:49,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:49,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53340 deadline: 1732301869806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:49,810 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:49,810 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:49,810 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:49,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:49,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53306 deadline: 1732301869806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:49,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:49,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:49,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:49,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-22T18:56:49,872 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=93 2024-11-22T18:56:49,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:49,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. as already flushing 2024-11-22T18:56:49,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:49,873 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] handler.RSProcedureHandler(58): pid=93 java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:49,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=93 java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:49,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=93 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:49,894 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/0d59d00df4fe4045b03595bebf119f3d 2024-11-22T18:56:49,902 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/9dda9b5d8e6b42aa9451651b661b7554 is 50, key is test_row_0/C:col10/1732301809144/Put/seqid=0 2024-11-22T18:56:49,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742122_1298 (size=12301) 2024-11-22T18:56:50,025 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=93 2024-11-22T18:56:50,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:50,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. as already flushing 2024-11-22T18:56:50,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:50,025 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=93}] handler.RSProcedureHandler(58): pid=93 java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:50,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=93 java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:50,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=93 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:50,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-22T18:56:50,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:50,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53228 deadline: 1732301870116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:50,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:50,118 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:50,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53250 deadline: 1732301870116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:50,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:50,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:50,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:50,119 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:50,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53324 deadline: 1732301870116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:50,119 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:50,119 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:50,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:50,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:50,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:50,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:50,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53340 deadline: 1732301870116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:50,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:50,119 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:50,119 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:50,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53306 deadline: 1732301870116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:50,119 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:50,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:50,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:50,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:50,178 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=93 2024-11-22T18:56:50,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:50,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. as already flushing 2024-11-22T18:56:50,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:50,179 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] handler.RSProcedureHandler(58): pid=93 java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:50,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=93 java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:50,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=93 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:56:50,307 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/9dda9b5d8e6b42aa9451651b661b7554 2024-11-22T18:56:50,311 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/d254867f8cde4f58ab72c0a1816596be as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/d254867f8cde4f58ab72c0a1816596be 2024-11-22T18:56:50,316 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/d254867f8cde4f58ab72c0a1816596be, entries=150, sequenceid=404, filesize=12.0 K 2024-11-22T18:56:50,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/0d59d00df4fe4045b03595bebf119f3d as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/0d59d00df4fe4045b03595bebf119f3d 2024-11-22T18:56:50,321 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/0d59d00df4fe4045b03595bebf119f3d, entries=150, sequenceid=404, filesize=12.0 K 2024-11-22T18:56:50,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/9dda9b5d8e6b42aa9451651b661b7554 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/9dda9b5d8e6b42aa9451651b661b7554 2024-11-22T18:56:50,326 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/9dda9b5d8e6b42aa9451651b661b7554, entries=150, sequenceid=404, filesize=12.0 K 2024-11-22T18:56:50,327 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for 2f5686bd89d51ddd0842891a5a86fd98 in 858ms, sequenceid=404, compaction requested=true 2024-11-22T18:56:50,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2f5686bd89d51ddd0842891a5a86fd98: 2024-11-22T18:56:50,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2f5686bd89d51ddd0842891a5a86fd98:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T18:56:50,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:56:50,327 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T18:56:50,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2f5686bd89d51ddd0842891a5a86fd98:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T18:56:50,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:56:50,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2f5686bd89d51ddd0842891a5a86fd98:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T18:56:50,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:56:50,327 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T18:56:50,330 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T18:56:50,330 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1541): 2f5686bd89d51ddd0842891a5a86fd98/B is initiating minor compaction (all files) 2024-11-22T18:56:50,330 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2f5686bd89d51ddd0842891a5a86fd98/B in TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:50,330 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/cf16dd29fa144f15ba7ba6c1a469c743, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/f751926882f54cca865e6e29935cbfe7, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/688a94ac74cf4525919648560fd0935c, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/0d59d00df4fe4045b03595bebf119f3d] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp, totalSize=48.8 K 2024-11-22T18:56:50,330 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T18:56:50,330 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1541): 2f5686bd89d51ddd0842891a5a86fd98/A is initiating minor compaction (all files) 2024-11-22T18:56:50,330 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2f5686bd89d51ddd0842891a5a86fd98/A in TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:50,330 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/f2ac0dfde571496db79625bf6ef4a82a, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/9cefde34d7b64fdcbb76fe085ca09e46, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/776bb2ac6ca747438b299ec15837dccd, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/d254867f8cde4f58ab72c0a1816596be] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp, totalSize=48.8 K 2024-11-22T18:56:50,330 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting cf16dd29fa144f15ba7ba6c1a469c743, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1732301806279 2024-11-22T18:56:50,331 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2ac0dfde571496db79625bf6ef4a82a, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1732301806279 2024-11-22T18:56:50,331 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting f751926882f54cca865e6e29935cbfe7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1732301806312 2024-11-22T18:56:50,331 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9cefde34d7b64fdcbb76fe085ca09e46, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1732301806312 2024-11-22T18:56:50,331 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 688a94ac74cf4525919648560fd0935c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1732301806949 2024-11-22T18:56:50,332 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d59d00df4fe4045b03595bebf119f3d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1732301809144 2024-11-22T18:56:50,332 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 776bb2ac6ca747438b299ec15837dccd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1732301806949 2024-11-22T18:56:50,332 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting d254867f8cde4f58ab72c0a1816596be, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1732301809144 2024-11-22T18:56:50,333 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=93 2024-11-22T18:56:50,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:50,333 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HRegion(2902): Flushing 2f5686bd89d51ddd0842891a5a86fd98 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-22T18:56:50,343 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2f5686bd89d51ddd0842891a5a86fd98#A#compaction#258 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:56:50,344 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/fbe57b731f3b4502adcf19b73e3f4bd1 is 50, key is test_row_0/A:col10/1732301809144/Put/seqid=0 2024-11-22T18:56:50,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-22T18:56:50,347 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2f5686bd89d51ddd0842891a5a86fd98#B#compaction#259 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:56:50,348 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/15fe1a1ddbe248eca661a6dcea4416f5 is 50, key is test_row_0/B:col10/1732301809144/Put/seqid=0 2024-11-22T18:56:50,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/4933ecf108234e02a8942eb0d53b73eb is 50, key is test_row_0/A:col10/1732301809478/Put/seqid=0 2024-11-22T18:56:50,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742125_1301 (size=12301) 2024-11-22T18:56:50,400 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/4933ecf108234e02a8942eb0d53b73eb 2024-11-22T18:56:50,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/efac42bca6814aff9d39b77c77fcadab is 50, key is test_row_0/B:col10/1732301809478/Put/seqid=0 2024-11-22T18:56:50,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742123_1299 (size=13255) 2024-11-22T18:56:50,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742126_1302 (size=12301) 2024-11-22T18:56:50,438 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/fbe57b731f3b4502adcf19b73e3f4bd1 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/fbe57b731f3b4502adcf19b73e3f4bd1 2024-11-22T18:56:50,445 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2f5686bd89d51ddd0842891a5a86fd98/A of 2f5686bd89d51ddd0842891a5a86fd98 into fbe57b731f3b4502adcf19b73e3f4bd1(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:56:50,445 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2f5686bd89d51ddd0842891a5a86fd98: 2024-11-22T18:56:50,446 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., storeName=2f5686bd89d51ddd0842891a5a86fd98/A, priority=12, startTime=1732301810327; duration=0sec 2024-11-22T18:56:50,447 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:56:50,447 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2f5686bd89d51ddd0842891a5a86fd98:A 2024-11-22T18:56:50,447 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T18:56:50,450 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T18:56:50,450 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1541): 2f5686bd89d51ddd0842891a5a86fd98/C is initiating minor compaction (all files) 2024-11-22T18:56:50,450 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2f5686bd89d51ddd0842891a5a86fd98/C in TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:50,450 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/4037cf78c7d84aafa28d43acbd987477, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/f9af0f753a26490c93810a4e78ea63dd, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/54713e67cdd241eb9632e0b4fe9ba71b, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/9dda9b5d8e6b42aa9451651b661b7554] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp, totalSize=48.8 K 2024-11-22T18:56:50,450 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4037cf78c7d84aafa28d43acbd987477, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1732301806279 2024-11-22T18:56:50,451 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9af0f753a26490c93810a4e78ea63dd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1732301806312 2024-11-22T18:56:50,451 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54713e67cdd241eb9632e0b4fe9ba71b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1732301806949 2024-11-22T18:56:50,453 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9dda9b5d8e6b42aa9451651b661b7554, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1732301809144 2024-11-22T18:56:50,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742124_1300 (size=13255) 2024-11-22T18:56:50,467 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2f5686bd89d51ddd0842891a5a86fd98#C#compaction#262 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:56:50,468 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/fe4844717cc146ae93a3d70dc0cb132b is 50, key is test_row_0/C:col10/1732301809144/Put/seqid=0 2024-11-22T18:56:50,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742127_1303 (size=13255) 2024-11-22T18:56:50,506 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/fe4844717cc146ae93a3d70dc0cb132b as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/fe4844717cc146ae93a3d70dc0cb132b 2024-11-22T18:56:50,522 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2f5686bd89d51ddd0842891a5a86fd98/C of 2f5686bd89d51ddd0842891a5a86fd98 into fe4844717cc146ae93a3d70dc0cb132b(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:56:50,523 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2f5686bd89d51ddd0842891a5a86fd98: 2024-11-22T18:56:50,523 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., storeName=2f5686bd89d51ddd0842891a5a86fd98/C, priority=12, startTime=1732301810327; duration=0sec 2024-11-22T18:56:50,523 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:56:50,523 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2f5686bd89d51ddd0842891a5a86fd98:C 2024-11-22T18:56:50,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on 2f5686bd89d51ddd0842891a5a86fd98 2024-11-22T18:56:50,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. as already flushing 2024-11-22T18:56:50,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:50,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53324 deadline: 1732301870652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:50,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:50,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:50,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:50,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:50,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53306 deadline: 1732301870653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:50,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:50,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:50,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:50,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:50,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53250 deadline: 1732301870653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:50,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:50,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53340 deadline: 1732301870654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:50,659 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:50,659 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:50,659 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:50,659 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:50,659 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:50,660 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:50,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:50,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53228 deadline: 1732301870658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:50,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:50,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:50,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:50,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:50,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53324 deadline: 1732301870765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:50,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:50,768 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:50,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53250 deadline: 1732301870765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:50,768 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:50,768 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:50,768 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:50,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:50,768 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:50,768 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:50,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53306 deadline: 1732301870765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:50,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:50,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:50,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:50,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:50,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53340 deadline: 1732301870766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:50,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:50,769 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:50,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53228 deadline: 1732301870766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:50,769 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:50,769 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:50,769 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:50,769 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:50,769 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:50,831 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/efac42bca6814aff9d39b77c77fcadab 2024-11-22T18:56:50,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/ecadcdbddc3249369bea12bc817d22f9 is 50, key is test_row_0/C:col10/1732301809478/Put/seqid=0 2024-11-22T18:56:50,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742128_1304 (size=12301) 2024-11-22T18:56:50,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-22T18:56:50,860 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/15fe1a1ddbe248eca661a6dcea4416f5 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/15fe1a1ddbe248eca661a6dcea4416f5 2024-11-22T18:56:50,864 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2f5686bd89d51ddd0842891a5a86fd98/B of 2f5686bd89d51ddd0842891a5a86fd98 into 15fe1a1ddbe248eca661a6dcea4416f5(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:56:50,864 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2f5686bd89d51ddd0842891a5a86fd98: 2024-11-22T18:56:50,865 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., storeName=2f5686bd89d51ddd0842891a5a86fd98/B, priority=12, startTime=1732301810327; duration=0sec 2024-11-22T18:56:50,865 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:56:50,865 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2f5686bd89d51ddd0842891a5a86fd98:B 2024-11-22T18:56:50,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:50,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53306 deadline: 1732301870976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:50,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:50,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:50,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:50,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:50,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53324 deadline: 1732301870976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:50,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:50,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53250 deadline: 1732301870976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:50,979 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:50,979 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:50,979 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:50,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:50,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53340 deadline: 1732301870976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:50,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:50,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:50,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:50,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:50,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53228 deadline: 1732301870976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:50,980 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:50,980 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:50,980 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:50,980 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:50,980 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:50,980 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:51,244 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/ecadcdbddc3249369bea12bc817d22f9 2024-11-22T18:56:51,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/4933ecf108234e02a8942eb0d53b73eb as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/4933ecf108234e02a8942eb0d53b73eb 2024-11-22T18:56:51,252 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/4933ecf108234e02a8942eb0d53b73eb, entries=150, sequenceid=415, filesize=12.0 K 2024-11-22T18:56:51,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/efac42bca6814aff9d39b77c77fcadab as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/efac42bca6814aff9d39b77c77fcadab 2024-11-22T18:56:51,256 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/efac42bca6814aff9d39b77c77fcadab, entries=150, sequenceid=415, filesize=12.0 K 2024-11-22T18:56:51,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/ecadcdbddc3249369bea12bc817d22f9 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/ecadcdbddc3249369bea12bc817d22f9 2024-11-22T18:56:51,261 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/ecadcdbddc3249369bea12bc817d22f9, entries=150, sequenceid=415, filesize=12.0 K 2024-11-22T18:56:51,262 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 2f5686bd89d51ddd0842891a5a86fd98 in 928ms, sequenceid=415, compaction requested=false 2024-11-22T18:56:51,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HRegion(2603): Flush status journal for 2f5686bd89d51ddd0842891a5a86fd98: 2024-11-22T18:56:51,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:51,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=93 2024-11-22T18:56:51,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster(4169): Remote procedure done, pid=93 2024-11-22T18:56:51,265 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-22T18:56:51,265 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5440 sec 2024-11-22T18:56:51,267 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=92, table=TestAcidGuarantees in 1.5480 sec 2024-11-22T18:56:51,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on 2f5686bd89d51ddd0842891a5a86fd98 2024-11-22T18:56:51,288 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T18:56:51,289 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2f5686bd89d51ddd0842891a5a86fd98 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-22T18:56:51,293 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/9e7d746253b2468985b285aef9f99055 is 50, key is test_row_0/A:col10/1732301810650/Put/seqid=0 2024-11-22T18:56:51,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:51,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:51,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53250 deadline: 1732301871292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:51,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53340 deadline: 1732301871292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:51,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:51,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:51,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:51,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:51,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:51,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:51,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742129_1305 (size=14741) 2024-11-22T18:56:51,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:51,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53306 deadline: 1732301871295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:51,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:51,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:51,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:51,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:51,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53324 deadline: 1732301871296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:51,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:51,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:51,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:51,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:51,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53228 deadline: 1732301871296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:51,300 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:51,300 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:51,300 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:51,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:51,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53250 deadline: 1732301871405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:51,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:51,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:51,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:51,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53306 deadline: 1732301871405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:51,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53340 deadline: 1732301871405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:51,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:51,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:51,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:51,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:51,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:51,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:51,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:51,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:51,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:51,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53228 deadline: 1732301871405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:51,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53324 deadline: 1732301871405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:51,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:51,409 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:51,409 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:51,409 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:51,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:51,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:51,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:51,569 INFO [Thread-1026 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1026. 2024-11-22T18:56:51,569 DEBUG [Thread-1026 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T18:56:51,569 DEBUG [Thread-1026 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:56:51,570 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T18:56:51,570 DEBUG [Thread-1026 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:56:51,571 INFO [Thread-1032 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1032. 2024-11-22T18:56:51,571 DEBUG [Thread-1032 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T18:56:51,571 DEBUG [Thread-1032 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:56:51,571 DEBUG [Thread-1032 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:56:51,571 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T18:56:51,571 INFO [Thread-1024 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1024. 2024-11-22T18:56:51,571 DEBUG [Thread-1024 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T18:56:51,571 DEBUG [Thread-1024 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:56:51,571 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T18:56:51,571 DEBUG [Thread-1024 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:56:51,573 INFO [Thread-1030 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1030. 2024-11-22T18:56:51,573 DEBUG [Thread-1030 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T18:56:51,574 DEBUG [Thread-1030 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:56:51,574 INFO [Thread-1028 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1028. 2024-11-22T18:56:51,574 DEBUG [Thread-1028 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T18:56:51,574 DEBUG [Thread-1030 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:56:51,574 DEBUG [Thread-1028 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:56:51,574 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T18:56:51,574 DEBUG [Thread-1028 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:56:51,574 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T18:56:51,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:51,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:51,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:51,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53250 deadline: 1732301871615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:51,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53228 deadline: 1732301871615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:51,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53306 deadline: 1732301871615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:51,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:51,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:51,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:51,616 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:51,616 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:51,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:51,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53324 deadline: 1732301871615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:51,616 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:51,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:51,616 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:51,616 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:51,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53340 deadline: 1732301871615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:51,616 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:51,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:51,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:51,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:51,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:51,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:51,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:51,699 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/9e7d746253b2468985b285aef9f99055 2024-11-22T18:56:51,705 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/b1759e668a134e5c81449d7aa21d9c96 is 50, key is test_row_0/B:col10/1732301810650/Put/seqid=0 2024-11-22T18:56:51,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742130_1306 (size=12301) 2024-11-22T18:56:51,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-22T18:56:51,866 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T18:56:51,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:51,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:51,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:51,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53250 deadline: 1732301871926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:51,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53228 deadline: 1732301871926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:51,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53306 deadline: 1732301871926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:51,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:51,927 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:51,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:51,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:51,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:51,927 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:51,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:51,927 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:51,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53340 deadline: 1732301871926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:51,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:51,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:51,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53324 deadline: 1732301871926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:51,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:51,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:51,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:51,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:51,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:51,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:51,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:52,109 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/b1759e668a134e5c81449d7aa21d9c96 2024-11-22T18:56:52,116 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/87170dd6a0954c36940435318e9d2c67 is 50, key is test_row_0/C:col10/1732301810650/Put/seqid=0 2024-11-22T18:56:52,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742131_1307 (size=12301) 2024-11-22T18:56:52,428 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-22T18:56:52,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:52,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:52,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:52,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53306 deadline: 1732301872436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:52,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53228 deadline: 1732301872436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:52,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53250 deadline: 1732301872436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:52,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:52,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:56:52,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53324 deadline: 1732301872436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:52,436 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:52,436 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:52,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:52,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53340 deadline: 1732301872436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:52,436 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:52,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:52,436 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:52,436 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:52,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:52,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:52,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:52,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:52,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:52,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:52,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:56:52,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:56:52,520 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/87170dd6a0954c36940435318e9d2c67 2024-11-22T18:56:52,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/9e7d746253b2468985b285aef9f99055 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/9e7d746253b2468985b285aef9f99055 2024-11-22T18:56:52,527 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/9e7d746253b2468985b285aef9f99055, entries=200, sequenceid=444, filesize=14.4 K 2024-11-22T18:56:52,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/b1759e668a134e5c81449d7aa21d9c96 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/b1759e668a134e5c81449d7aa21d9c96 2024-11-22T18:56:52,531 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/b1759e668a134e5c81449d7aa21d9c96, entries=150, sequenceid=444, filesize=12.0 K 2024-11-22T18:56:52,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/87170dd6a0954c36940435318e9d2c67 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/87170dd6a0954c36940435318e9d2c67 2024-11-22T18:56:52,534 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/87170dd6a0954c36940435318e9d2c67, entries=150, sequenceid=444, filesize=12.0 K 2024-11-22T18:56:52,535 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 2f5686bd89d51ddd0842891a5a86fd98 in 1247ms, sequenceid=444, compaction requested=true 2024-11-22T18:56:52,535 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2f5686bd89d51ddd0842891a5a86fd98: 2024-11-22T18:56:52,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2f5686bd89d51ddd0842891a5a86fd98:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T18:56:52,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:56:52,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2f5686bd89d51ddd0842891a5a86fd98:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T18:56:52,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:56:52,535 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T18:56:52,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2f5686bd89d51ddd0842891a5a86fd98:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T18:56:52,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:56:52,535 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T18:56:52,536 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40297 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T18:56:52,536 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T18:56:52,536 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1541): 2f5686bd89d51ddd0842891a5a86fd98/A is initiating minor compaction (all files) 2024-11-22T18:56:52,536 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1541): 2f5686bd89d51ddd0842891a5a86fd98/B is initiating minor compaction (all files) 2024-11-22T18:56:52,536 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2f5686bd89d51ddd0842891a5a86fd98/A in TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:52,536 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2f5686bd89d51ddd0842891a5a86fd98/B in TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:52,536 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/fbe57b731f3b4502adcf19b73e3f4bd1, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/4933ecf108234e02a8942eb0d53b73eb, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/9e7d746253b2468985b285aef9f99055] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp, totalSize=39.4 K 2024-11-22T18:56:52,536 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/15fe1a1ddbe248eca661a6dcea4416f5, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/efac42bca6814aff9d39b77c77fcadab, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/b1759e668a134e5c81449d7aa21d9c96] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp, totalSize=37.0 K 2024-11-22T18:56:52,536 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting fbe57b731f3b4502adcf19b73e3f4bd1, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1732301809144 2024-11-22T18:56:52,536 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 15fe1a1ddbe248eca661a6dcea4416f5, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1732301809144 2024-11-22T18:56:52,537 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4933ecf108234e02a8942eb0d53b73eb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1732301809475 2024-11-22T18:56:52,537 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting efac42bca6814aff9d39b77c77fcadab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1732301809475 2024-11-22T18:56:52,537 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e7d746253b2468985b285aef9f99055, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1732301810650 2024-11-22T18:56:52,537 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting b1759e668a134e5c81449d7aa21d9c96, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1732301810650 2024-11-22T18:56:52,543 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2f5686bd89d51ddd0842891a5a86fd98#B#compaction#268 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:56:52,543 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2f5686bd89d51ddd0842891a5a86fd98#A#compaction#267 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:56:52,543 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/7da54b79c0fa4624bfa98babc055eec4 is 50, key is test_row_0/B:col10/1732301810650/Put/seqid=0 2024-11-22T18:56:52,543 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/148eab2460334f9ba4cceaefd7679ab2 is 50, key is test_row_0/A:col10/1732301810650/Put/seqid=0 2024-11-22T18:56:52,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742132_1308 (size=13357) 2024-11-22T18:56:52,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742133_1309 (size=13357) 2024-11-22T18:56:52,951 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/7da54b79c0fa4624bfa98babc055eec4 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/7da54b79c0fa4624bfa98babc055eec4 2024-11-22T18:56:52,952 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/148eab2460334f9ba4cceaefd7679ab2 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/148eab2460334f9ba4cceaefd7679ab2 2024-11-22T18:56:52,956 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2f5686bd89d51ddd0842891a5a86fd98/B of 2f5686bd89d51ddd0842891a5a86fd98 into 7da54b79c0fa4624bfa98babc055eec4(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:56:52,956 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2f5686bd89d51ddd0842891a5a86fd98: 2024-11-22T18:56:52,956 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., storeName=2f5686bd89d51ddd0842891a5a86fd98/B, priority=13, startTime=1732301812535; duration=0sec 2024-11-22T18:56:52,956 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:56:52,956 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2f5686bd89d51ddd0842891a5a86fd98:B 2024-11-22T18:56:52,956 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T18:56:52,956 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2f5686bd89d51ddd0842891a5a86fd98/A of 2f5686bd89d51ddd0842891a5a86fd98 into 148eab2460334f9ba4cceaefd7679ab2(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:56:52,956 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2f5686bd89d51ddd0842891a5a86fd98: 2024-11-22T18:56:52,956 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., storeName=2f5686bd89d51ddd0842891a5a86fd98/A, priority=13, startTime=1732301812535; duration=0sec 2024-11-22T18:56:52,956 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:56:52,956 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2f5686bd89d51ddd0842891a5a86fd98:A 2024-11-22T18:56:52,957 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T18:56:52,957 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1541): 2f5686bd89d51ddd0842891a5a86fd98/C is initiating minor compaction (all files) 2024-11-22T18:56:52,957 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2f5686bd89d51ddd0842891a5a86fd98/C in TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:52,957 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/fe4844717cc146ae93a3d70dc0cb132b, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/ecadcdbddc3249369bea12bc817d22f9, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/87170dd6a0954c36940435318e9d2c67] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp, totalSize=37.0 K 2024-11-22T18:56:52,957 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting fe4844717cc146ae93a3d70dc0cb132b, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1732301809144 2024-11-22T18:56:52,957 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting ecadcdbddc3249369bea12bc817d22f9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1732301809475 2024-11-22T18:56:52,958 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 87170dd6a0954c36940435318e9d2c67, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1732301810650 2024-11-22T18:56:52,963 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2f5686bd89d51ddd0842891a5a86fd98#C#compaction#269 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:56:52,963 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/84a36a1491034585a91ea05bf3c5f36d is 50, key is test_row_0/C:col10/1732301810650/Put/seqid=0 2024-11-22T18:56:52,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742134_1310 (size=13357) 2024-11-22T18:56:53,371 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/84a36a1491034585a91ea05bf3c5f36d as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/84a36a1491034585a91ea05bf3c5f36d 2024-11-22T18:56:53,375 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2f5686bd89d51ddd0842891a5a86fd98/C of 2f5686bd89d51ddd0842891a5a86fd98 into 84a36a1491034585a91ea05bf3c5f36d(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:56:53,375 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2f5686bd89d51ddd0842891a5a86fd98: 2024-11-22T18:56:53,375 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., storeName=2f5686bd89d51ddd0842891a5a86fd98/C, priority=13, startTime=1732301812535; duration=0sec 2024-11-22T18:56:53,375 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:56:53,375 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2f5686bd89d51ddd0842891a5a86fd98:C 2024-11-22T18:56:53,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on 2f5686bd89d51ddd0842891a5a86fd98 2024-11-22T18:56:53,447 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T18:56:53,447 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2f5686bd89d51ddd0842891a5a86fd98 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-22T18:56:53,447 INFO [Thread-1021 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1021. 2024-11-22T18:56:53,447 DEBUG [Thread-1021 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T18:56:53,447 INFO [Thread-1017 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1017. 2024-11-22T18:56:53,447 INFO [Thread-1013 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1013. 2024-11-22T18:56:53,447 DEBUG [Thread-1021 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:56:53,447 DEBUG [Thread-1013 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T18:56:53,447 DEBUG [Thread-1017 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T18:56:53,447 DEBUG [Thread-1013 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:56:53,447 DEBUG [Thread-1017 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:56:53,448 DEBUG [Thread-1021 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:56:53,448 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T18:56:53,448 DEBUG [Thread-1013 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:56:53,448 DEBUG [Thread-1017 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:56:53,448 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T18:56:53,448 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T18:56:53,449 INFO [Thread-1019 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1019. 2024-11-22T18:56:53,449 DEBUG [Thread-1019 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T18:56:53,449 DEBUG [Thread-1019 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:56:53,449 DEBUG [Thread-1019 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:56:53,449 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T18:56:53,450 INFO [Thread-1015 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1015. 2024-11-22T18:56:53,450 DEBUG [Thread-1015 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T18:56:53,450 DEBUG [Thread-1015 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:56:53,450 DEBUG [Thread-1015 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:56:53,450 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T18:56:53,450 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-22T18:56:53,450 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 78 2024-11-22T18:56:53,450 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 75 2024-11-22T18:56:53,450 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 69 2024-11-22T18:56:53,450 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 72 2024-11-22T18:56:53,450 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 68 2024-11-22T18:56:53,450 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-22T18:56:53,450 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3542 2024-11-22T18:56:53,450 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3755 2024-11-22T18:56:53,450 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3646 2024-11-22T18:56:53,450 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3551 2024-11-22T18:56:53,450 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3745 2024-11-22T18:56:53,450 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-22T18:56:53,450 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-22T18:56:53,450 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T18:56:53,451 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:56:53,451 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:56:53,451 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T18:56:53,451 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-22T18:56:53,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] procedure2.ProcedureExecutor(1139): Stored pid=94, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-22T18:56:53,453 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/4a198b552cb8438eafc2c9aab888d5f5 is 50, key is test_row_0/A:col10/1732301813446/Put/seqid=0 2024-11-22T18:56:53,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-22T18:56:53,455 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732301813455"}]},"ts":"1732301813455"} 2024-11-22T18:56:53,456 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-22T18:56:53,456 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-22T18:56:53,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742135_1311 (size=12301) 2024-11-22T18:56:53,457 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-22T18:56:53,458 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2f5686bd89d51ddd0842891a5a86fd98, UNASSIGN}] 2024-11-22T18:56:53,458 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2f5686bd89d51ddd0842891a5a86fd98, UNASSIGN 2024-11-22T18:56:53,459 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=2f5686bd89d51ddd0842891a5a86fd98, regionState=CLOSING, regionLocation=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:53,460 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2f5686bd89d51ddd0842891a5a86fd98, UNASSIGN because future has completed 2024-11-22T18:56:53,460 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-22T18:56:53,460 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=97, ppid=96, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173}] 2024-11-22T18:56:53,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-22T18:56:53,612 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(122): Close 2f5686bd89d51ddd0842891a5a86fd98 2024-11-22T18:56:53,612 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-22T18:56:53,612 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1722): Closing 2f5686bd89d51ddd0842891a5a86fd98, disabling compactions & flushes 2024-11-22T18:56:53,613 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:53,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-22T18:56:53,857 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/4a198b552cb8438eafc2c9aab888d5f5 2024-11-22T18:56:53,863 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/cc2a0ee238d44250995857503366fabd is 50, key is test_row_0/B:col10/1732301813446/Put/seqid=0 2024-11-22T18:56:53,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742136_1312 (size=12301) 2024-11-22T18:56:54,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-22T18:56:54,267 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/cc2a0ee238d44250995857503366fabd 2024-11-22T18:56:54,274 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/36c4b16a26ae427eb12b6d47a371aa53 is 50, key is test_row_0/C:col10/1732301813446/Put/seqid=0 2024-11-22T18:56:54,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742137_1313 (size=12301) 2024-11-22T18:56:54,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-22T18:56:54,679 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/36c4b16a26ae427eb12b6d47a371aa53 2024-11-22T18:56:54,683 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/4a198b552cb8438eafc2c9aab888d5f5 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/4a198b552cb8438eafc2c9aab888d5f5 2024-11-22T18:56:54,686 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/4a198b552cb8438eafc2c9aab888d5f5, entries=150, sequenceid=461, filesize=12.0 K 2024-11-22T18:56:54,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/cc2a0ee238d44250995857503366fabd as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/cc2a0ee238d44250995857503366fabd 2024-11-22T18:56:54,689 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/cc2a0ee238d44250995857503366fabd, entries=150, sequenceid=461, filesize=12.0 K 2024-11-22T18:56:54,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/36c4b16a26ae427eb12b6d47a371aa53 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/36c4b16a26ae427eb12b6d47a371aa53 2024-11-22T18:56:54,692 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/36c4b16a26ae427eb12b6d47a371aa53, entries=150, sequenceid=461, filesize=12.0 K 2024-11-22T18:56:54,693 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=6.71 KB/6870 for 2f5686bd89d51ddd0842891a5a86fd98 in 1246ms, sequenceid=461, compaction requested=false 2024-11-22T18:56:54,693 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2f5686bd89d51ddd0842891a5a86fd98: 2024-11-22T18:56:54,693 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:54,693 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:54,693 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. after waiting 0 ms 2024-11-22T18:56:54,693 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:54,693 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(2902): Flushing 2f5686bd89d51ddd0842891a5a86fd98 3/3 column families, dataSize=6.71 KB heapSize=18.33 KB 2024-11-22T18:56:54,697 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/333190869edb4634bcbb32f8ab0f49be is 50, key is test_row_1/A:col10/1732301813449/Put/seqid=0 2024-11-22T18:56:54,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742138_1314 (size=7415) 2024-11-22T18:56:55,100 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/333190869edb4634bcbb32f8ab0f49be 2024-11-22T18:56:55,107 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/80a57c014e6c43c79bdc0d2df9e9af13 is 50, key is test_row_1/B:col10/1732301813449/Put/seqid=0 2024-11-22T18:56:55,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742139_1315 (size=7415) 2024-11-22T18:56:55,511 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/80a57c014e6c43c79bdc0d2df9e9af13 2024-11-22T18:56:55,517 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/34d8b7aaca054ca5b4cd547346e5c426 is 50, key is test_row_1/C:col10/1732301813449/Put/seqid=0 2024-11-22T18:56:55,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742140_1316 (size=7415) 2024-11-22T18:56:55,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-22T18:56:55,921 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/34d8b7aaca054ca5b4cd547346e5c426 2024-11-22T18:56:55,925 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/A/333190869edb4634bcbb32f8ab0f49be as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/333190869edb4634bcbb32f8ab0f49be 2024-11-22T18:56:55,928 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/333190869edb4634bcbb32f8ab0f49be, entries=50, sequenceid=465, filesize=7.2 K 2024-11-22T18:56:55,928 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/B/80a57c014e6c43c79bdc0d2df9e9af13 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/80a57c014e6c43c79bdc0d2df9e9af13 2024-11-22T18:56:55,931 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/80a57c014e6c43c79bdc0d2df9e9af13, entries=50, sequenceid=465, filesize=7.2 K 2024-11-22T18:56:55,932 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/.tmp/C/34d8b7aaca054ca5b4cd547346e5c426 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/34d8b7aaca054ca5b4cd547346e5c426 2024-11-22T18:56:55,935 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/34d8b7aaca054ca5b4cd547346e5c426, entries=50, sequenceid=465, filesize=7.2 K 2024-11-22T18:56:55,936 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(3140): Finished flush of dataSize ~6.71 KB/6870, heapSize ~18.28 KB/18720, currentSize=0 B/0 for 2f5686bd89d51ddd0842891a5a86fd98 in 1242ms, sequenceid=465, compaction requested=true 2024-11-22T18:56:55,936 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/06009c221f8c4d44b7023b2016014280, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/9d15795c7b624832b98a6871760e268d, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/0b154bac2b4b4b05a5054ad4399fd0f6, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/31dbc548ced345a3bcb78de1b87d1830, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/1b78062a195147188cf62db5d20ac9fc, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/a17ebd37affb40f1b3b2a8cb4d8681cd, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/15a3680435b447abb5ff5f23474b001e, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/9326c308691e4be5a5f7f1dd0612e52e, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/cef35b07137e4d469e33dc1d7f89dbc2, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/1bef9a3b4b6f4f0e99928ea8055dc581, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/143dea4cac924dea8772b75350cf6a4f, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/3ba2b8e78db04582a108de6e0c35c2ff, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/ce9d5c51e33e441f891c18eec9a628d8, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/a7e10a7e98fb46c89694e1d36c900ed5, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/85df1074a5f048319e6fa8cf44927d39, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/3d65d24a9e8b4e66a5b2dad47f74e4dd, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/1a5b5b12ab4d44f9bdad3e8b60b58714, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/b6b7585abb594e11b658c8d248e7948f, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/0632dde5db2c4d4d98c8e8bae04b6458, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/f02b337c54be4b54bfe4c0208fa4efdd, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/f9f428bfdd0940d5b574872f78b4a42f, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/36d452d0b4fe4f8297b9af6297a7ed50, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/ce57e715135e4f0ebe94536d83c4b370, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/f2ac0dfde571496db79625bf6ef4a82a, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/22e0685c9db04905a0d5996eb43ffa6b, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/9cefde34d7b64fdcbb76fe085ca09e46, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/776bb2ac6ca747438b299ec15837dccd, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/fbe57b731f3b4502adcf19b73e3f4bd1, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/d254867f8cde4f58ab72c0a1816596be, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/4933ecf108234e02a8942eb0d53b73eb, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/9e7d746253b2468985b285aef9f99055] to archive 2024-11-22T18:56:55,937 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T18:56:55,938 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/06009c221f8c4d44b7023b2016014280 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/06009c221f8c4d44b7023b2016014280 2024-11-22T18:56:55,939 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/9d15795c7b624832b98a6871760e268d to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/9d15795c7b624832b98a6871760e268d 2024-11-22T18:56:55,940 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/0b154bac2b4b4b05a5054ad4399fd0f6 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/0b154bac2b4b4b05a5054ad4399fd0f6 2024-11-22T18:56:55,941 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/31dbc548ced345a3bcb78de1b87d1830 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/31dbc548ced345a3bcb78de1b87d1830 2024-11-22T18:56:55,942 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/1b78062a195147188cf62db5d20ac9fc to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/1b78062a195147188cf62db5d20ac9fc 2024-11-22T18:56:55,942 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/a17ebd37affb40f1b3b2a8cb4d8681cd to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/a17ebd37affb40f1b3b2a8cb4d8681cd 2024-11-22T18:56:55,943 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/15a3680435b447abb5ff5f23474b001e to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/15a3680435b447abb5ff5f23474b001e 2024-11-22T18:56:55,944 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/9326c308691e4be5a5f7f1dd0612e52e to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/9326c308691e4be5a5f7f1dd0612e52e 2024-11-22T18:56:55,945 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/cef35b07137e4d469e33dc1d7f89dbc2 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/cef35b07137e4d469e33dc1d7f89dbc2 2024-11-22T18:56:55,946 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/1bef9a3b4b6f4f0e99928ea8055dc581 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/1bef9a3b4b6f4f0e99928ea8055dc581 2024-11-22T18:56:55,947 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/143dea4cac924dea8772b75350cf6a4f to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/143dea4cac924dea8772b75350cf6a4f 2024-11-22T18:56:55,948 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/3ba2b8e78db04582a108de6e0c35c2ff to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/3ba2b8e78db04582a108de6e0c35c2ff 2024-11-22T18:56:55,949 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/ce9d5c51e33e441f891c18eec9a628d8 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/ce9d5c51e33e441f891c18eec9a628d8 2024-11-22T18:56:55,949 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/a7e10a7e98fb46c89694e1d36c900ed5 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/a7e10a7e98fb46c89694e1d36c900ed5 2024-11-22T18:56:55,950 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/85df1074a5f048319e6fa8cf44927d39 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/85df1074a5f048319e6fa8cf44927d39 2024-11-22T18:56:55,951 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/3d65d24a9e8b4e66a5b2dad47f74e4dd to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/3d65d24a9e8b4e66a5b2dad47f74e4dd 2024-11-22T18:56:55,952 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/1a5b5b12ab4d44f9bdad3e8b60b58714 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/1a5b5b12ab4d44f9bdad3e8b60b58714 2024-11-22T18:56:55,953 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/b6b7585abb594e11b658c8d248e7948f to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/b6b7585abb594e11b658c8d248e7948f 2024-11-22T18:56:55,954 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/0632dde5db2c4d4d98c8e8bae04b6458 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/0632dde5db2c4d4d98c8e8bae04b6458 2024-11-22T18:56:55,955 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/f02b337c54be4b54bfe4c0208fa4efdd to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/f02b337c54be4b54bfe4c0208fa4efdd 2024-11-22T18:56:55,956 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/f9f428bfdd0940d5b574872f78b4a42f to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/f9f428bfdd0940d5b574872f78b4a42f 2024-11-22T18:56:55,957 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/36d452d0b4fe4f8297b9af6297a7ed50 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/36d452d0b4fe4f8297b9af6297a7ed50 2024-11-22T18:56:55,958 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/ce57e715135e4f0ebe94536d83c4b370 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/ce57e715135e4f0ebe94536d83c4b370 2024-11-22T18:56:55,958 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/f2ac0dfde571496db79625bf6ef4a82a to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/f2ac0dfde571496db79625bf6ef4a82a 2024-11-22T18:56:55,959 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/22e0685c9db04905a0d5996eb43ffa6b to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/22e0685c9db04905a0d5996eb43ffa6b 2024-11-22T18:56:55,960 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/9cefde34d7b64fdcbb76fe085ca09e46 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/9cefde34d7b64fdcbb76fe085ca09e46 2024-11-22T18:56:55,961 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/776bb2ac6ca747438b299ec15837dccd to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/776bb2ac6ca747438b299ec15837dccd 2024-11-22T18:56:55,962 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/fbe57b731f3b4502adcf19b73e3f4bd1 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/fbe57b731f3b4502adcf19b73e3f4bd1 2024-11-22T18:56:55,963 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/d254867f8cde4f58ab72c0a1816596be to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/d254867f8cde4f58ab72c0a1816596be 2024-11-22T18:56:55,963 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/4933ecf108234e02a8942eb0d53b73eb to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/4933ecf108234e02a8942eb0d53b73eb 2024-11-22T18:56:55,964 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/9e7d746253b2468985b285aef9f99055 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/9e7d746253b2468985b285aef9f99055 2024-11-22T18:56:55,965 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/5a4c214d15854124a0a1b469ea019b16, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/244b3346b86b49eab9a34443228e6acd, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/7416a47e82f046219a6c76451efc1791, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/c34952d9476d4a0d851618c24f7c431b, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/d346e0e93b534140a95ee56c35bfde22, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/c21e1337b0e64a409a637c7f640f880f, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/1e95f46f581d445ab625c7a1d3e23f69, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/f2745634300e44718cccd86844308c12, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/40002127af134b6da1d2cb67b29eb8c9, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/a465780d00df427eae97c4e4c9a70c19, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/2cb4118a057e48bf9ad85b79f0f0227f, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/46a8f9ba1812476ca04badd77f2e1677, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/e5c98c63cab741459a3e42b5eb5e4aaf, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/e6af29262ff84900993340a58d392b49, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/bca59a43951c4cb286aebcfbf4054ca5, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/24263897fe244fd0b154156039cad35c, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/73614efc75d14a58ade63244eadd11a0, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/9803899540d44ca8892da796c6077051, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/e50ddf21aead498285b2e8207715d0d3, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/2734729b21c94a29b408f828e4eabe4a, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/20d88c7e0e634381b08c3ac56b871044, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/75b0c112fc0b4b45a357fa90fde33e03, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/b7a34ecc207047d3875cb5a2d41c316c, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/cf16dd29fa144f15ba7ba6c1a469c743, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/473a825efbcb4b33862a66024acb525f, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/f751926882f54cca865e6e29935cbfe7, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/688a94ac74cf4525919648560fd0935c, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/15fe1a1ddbe248eca661a6dcea4416f5, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/0d59d00df4fe4045b03595bebf119f3d, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/efac42bca6814aff9d39b77c77fcadab, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/b1759e668a134e5c81449d7aa21d9c96] to archive 2024-11-22T18:56:55,966 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T18:56:55,967 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/5a4c214d15854124a0a1b469ea019b16 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/5a4c214d15854124a0a1b469ea019b16 2024-11-22T18:56:55,969 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/244b3346b86b49eab9a34443228e6acd to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/244b3346b86b49eab9a34443228e6acd 2024-11-22T18:56:55,970 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/7416a47e82f046219a6c76451efc1791 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/7416a47e82f046219a6c76451efc1791 2024-11-22T18:56:55,971 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/c34952d9476d4a0d851618c24f7c431b to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/c34952d9476d4a0d851618c24f7c431b 2024-11-22T18:56:55,972 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/d346e0e93b534140a95ee56c35bfde22 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/d346e0e93b534140a95ee56c35bfde22 2024-11-22T18:56:55,973 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/c21e1337b0e64a409a637c7f640f880f to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/c21e1337b0e64a409a637c7f640f880f 2024-11-22T18:56:55,974 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/1e95f46f581d445ab625c7a1d3e23f69 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/1e95f46f581d445ab625c7a1d3e23f69 2024-11-22T18:56:55,975 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/f2745634300e44718cccd86844308c12 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/f2745634300e44718cccd86844308c12 2024-11-22T18:56:55,976 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/40002127af134b6da1d2cb67b29eb8c9 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/40002127af134b6da1d2cb67b29eb8c9 2024-11-22T18:56:55,977 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/a465780d00df427eae97c4e4c9a70c19 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/a465780d00df427eae97c4e4c9a70c19 2024-11-22T18:56:55,978 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/2cb4118a057e48bf9ad85b79f0f0227f to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/2cb4118a057e48bf9ad85b79f0f0227f 2024-11-22T18:56:55,979 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/46a8f9ba1812476ca04badd77f2e1677 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/46a8f9ba1812476ca04badd77f2e1677 2024-11-22T18:56:55,980 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/e5c98c63cab741459a3e42b5eb5e4aaf to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/e5c98c63cab741459a3e42b5eb5e4aaf 2024-11-22T18:56:55,981 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/e6af29262ff84900993340a58d392b49 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/e6af29262ff84900993340a58d392b49 2024-11-22T18:56:55,982 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/bca59a43951c4cb286aebcfbf4054ca5 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/bca59a43951c4cb286aebcfbf4054ca5 2024-11-22T18:56:55,983 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/24263897fe244fd0b154156039cad35c to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/24263897fe244fd0b154156039cad35c 2024-11-22T18:56:55,984 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/73614efc75d14a58ade63244eadd11a0 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/73614efc75d14a58ade63244eadd11a0 2024-11-22T18:56:55,985 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/9803899540d44ca8892da796c6077051 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/9803899540d44ca8892da796c6077051 2024-11-22T18:56:55,986 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/e50ddf21aead498285b2e8207715d0d3 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/e50ddf21aead498285b2e8207715d0d3 2024-11-22T18:56:55,987 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/2734729b21c94a29b408f828e4eabe4a to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/2734729b21c94a29b408f828e4eabe4a 2024-11-22T18:56:55,988 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/20d88c7e0e634381b08c3ac56b871044 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/20d88c7e0e634381b08c3ac56b871044 2024-11-22T18:56:55,989 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/75b0c112fc0b4b45a357fa90fde33e03 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/75b0c112fc0b4b45a357fa90fde33e03 2024-11-22T18:56:55,990 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/b7a34ecc207047d3875cb5a2d41c316c to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/b7a34ecc207047d3875cb5a2d41c316c 2024-11-22T18:56:55,991 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/cf16dd29fa144f15ba7ba6c1a469c743 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/cf16dd29fa144f15ba7ba6c1a469c743 2024-11-22T18:56:55,992 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/473a825efbcb4b33862a66024acb525f to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/473a825efbcb4b33862a66024acb525f 2024-11-22T18:56:55,993 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/f751926882f54cca865e6e29935cbfe7 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/f751926882f54cca865e6e29935cbfe7 2024-11-22T18:56:55,994 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/688a94ac74cf4525919648560fd0935c to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/688a94ac74cf4525919648560fd0935c 2024-11-22T18:56:55,995 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/15fe1a1ddbe248eca661a6dcea4416f5 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/15fe1a1ddbe248eca661a6dcea4416f5 2024-11-22T18:56:55,996 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/0d59d00df4fe4045b03595bebf119f3d to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/0d59d00df4fe4045b03595bebf119f3d 2024-11-22T18:56:55,998 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/efac42bca6814aff9d39b77c77fcadab to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/efac42bca6814aff9d39b77c77fcadab 2024-11-22T18:56:55,999 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/b1759e668a134e5c81449d7aa21d9c96 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/b1759e668a134e5c81449d7aa21d9c96 2024-11-22T18:56:56,000 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/6afe73a7d389494ea0ace9014167ec72, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/b61b5303ad6f4be8b4c79b0315592f30, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/9be18a8b9c6a4e3096b6c85b94e95710, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/77f983792edd4b2c94aadbba1d826681, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/1d7c60bceb0741d7b4fb1646c7f73e32, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/df8f157cc2904766a7fc2ba3e4894f43, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/9e9955c5798b4d8bb378a9a53a1972c2, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/2f1c3b2174ae4653917041db1eacf4f6, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/85560fa7a98c452780a6942085f2900e, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/cdf2e731c4d5473dae37f706a44e959d, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/31340e9b41114f45a9d3d8eac7f43d1f, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/271cd2a7f3684da4bfdcebc6454fbf61, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/ac859694f8714fd9bdd1726188d2bcd3, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/41dac32379f645b19829d3b92a4e960b, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/de180626aede4534a0662cd27b6134c9, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/c996fb88b9ec4f17bbc8ec9a70eb319a, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/5e8a1b53c9c445cbb6b39aff26a5f239, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/37c930a424f24de8b402750f71961501, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/145ffba30ecf4067a9f1450e83ff8ba9, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/6491683126464d55b71b719290edbf63, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/cafa9dad25c741189b2352a7b2073bfe, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/654d713dce88401a9d63cd178464f81b, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/b6360ca7cf334a49bc3f3d368deb53ce, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/4037cf78c7d84aafa28d43acbd987477, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/e539b75cda9649dc998259c12bc992c5, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/f9af0f753a26490c93810a4e78ea63dd, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/54713e67cdd241eb9632e0b4fe9ba71b, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/fe4844717cc146ae93a3d70dc0cb132b, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/9dda9b5d8e6b42aa9451651b661b7554, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/ecadcdbddc3249369bea12bc817d22f9, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/87170dd6a0954c36940435318e9d2c67] to archive 2024-11-22T18:56:56,001 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T18:56:56,002 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/6afe73a7d389494ea0ace9014167ec72 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/6afe73a7d389494ea0ace9014167ec72 2024-11-22T18:56:56,003 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/b61b5303ad6f4be8b4c79b0315592f30 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/b61b5303ad6f4be8b4c79b0315592f30 2024-11-22T18:56:56,004 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/9be18a8b9c6a4e3096b6c85b94e95710 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/9be18a8b9c6a4e3096b6c85b94e95710 2024-11-22T18:56:56,005 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/77f983792edd4b2c94aadbba1d826681 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/77f983792edd4b2c94aadbba1d826681 2024-11-22T18:56:56,006 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/1d7c60bceb0741d7b4fb1646c7f73e32 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/1d7c60bceb0741d7b4fb1646c7f73e32 2024-11-22T18:56:56,007 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/df8f157cc2904766a7fc2ba3e4894f43 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/df8f157cc2904766a7fc2ba3e4894f43 2024-11-22T18:56:56,008 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/9e9955c5798b4d8bb378a9a53a1972c2 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/9e9955c5798b4d8bb378a9a53a1972c2 2024-11-22T18:56:56,009 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/2f1c3b2174ae4653917041db1eacf4f6 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/2f1c3b2174ae4653917041db1eacf4f6 2024-11-22T18:56:56,010 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/85560fa7a98c452780a6942085f2900e to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/85560fa7a98c452780a6942085f2900e 2024-11-22T18:56:56,010 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/cdf2e731c4d5473dae37f706a44e959d to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/cdf2e731c4d5473dae37f706a44e959d 2024-11-22T18:56:56,011 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/31340e9b41114f45a9d3d8eac7f43d1f to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/31340e9b41114f45a9d3d8eac7f43d1f 2024-11-22T18:56:56,013 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/271cd2a7f3684da4bfdcebc6454fbf61 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/271cd2a7f3684da4bfdcebc6454fbf61 2024-11-22T18:56:56,013 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/ac859694f8714fd9bdd1726188d2bcd3 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/ac859694f8714fd9bdd1726188d2bcd3 2024-11-22T18:56:56,014 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/41dac32379f645b19829d3b92a4e960b to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/41dac32379f645b19829d3b92a4e960b 2024-11-22T18:56:56,015 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/de180626aede4534a0662cd27b6134c9 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/de180626aede4534a0662cd27b6134c9 2024-11-22T18:56:56,016 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/c996fb88b9ec4f17bbc8ec9a70eb319a to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/c996fb88b9ec4f17bbc8ec9a70eb319a 2024-11-22T18:56:56,017 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/5e8a1b53c9c445cbb6b39aff26a5f239 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/5e8a1b53c9c445cbb6b39aff26a5f239 2024-11-22T18:56:56,017 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/37c930a424f24de8b402750f71961501 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/37c930a424f24de8b402750f71961501 2024-11-22T18:56:56,018 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/145ffba30ecf4067a9f1450e83ff8ba9 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/145ffba30ecf4067a9f1450e83ff8ba9 2024-11-22T18:56:56,019 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/6491683126464d55b71b719290edbf63 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/6491683126464d55b71b719290edbf63 2024-11-22T18:56:56,020 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/cafa9dad25c741189b2352a7b2073bfe to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/cafa9dad25c741189b2352a7b2073bfe 2024-11-22T18:56:56,021 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/654d713dce88401a9d63cd178464f81b to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/654d713dce88401a9d63cd178464f81b 2024-11-22T18:56:56,022 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/b6360ca7cf334a49bc3f3d368deb53ce to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/b6360ca7cf334a49bc3f3d368deb53ce 2024-11-22T18:56:56,023 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/4037cf78c7d84aafa28d43acbd987477 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/4037cf78c7d84aafa28d43acbd987477 2024-11-22T18:56:56,024 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/e539b75cda9649dc998259c12bc992c5 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/e539b75cda9649dc998259c12bc992c5 2024-11-22T18:56:56,024 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/f9af0f753a26490c93810a4e78ea63dd to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/f9af0f753a26490c93810a4e78ea63dd 2024-11-22T18:56:56,025 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/54713e67cdd241eb9632e0b4fe9ba71b to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/54713e67cdd241eb9632e0b4fe9ba71b 2024-11-22T18:56:56,026 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/fe4844717cc146ae93a3d70dc0cb132b to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/fe4844717cc146ae93a3d70dc0cb132b 2024-11-22T18:56:56,027 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/9dda9b5d8e6b42aa9451651b661b7554 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/9dda9b5d8e6b42aa9451651b661b7554 2024-11-22T18:56:56,028 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/ecadcdbddc3249369bea12bc817d22f9 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/ecadcdbddc3249369bea12bc817d22f9 2024-11-22T18:56:56,029 DEBUG [StoreCloser-TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/87170dd6a0954c36940435318e9d2c67 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/87170dd6a0954c36940435318e9d2c67 2024-11-22T18:56:56,033 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/recovered.edits/468.seqid, newMaxSeqId=468, maxSeqId=1 2024-11-22T18:56:56,033 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. 2024-11-22T18:56:56,033 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1676): Region close journal for 2f5686bd89d51ddd0842891a5a86fd98: Waiting for close lock at 1732301813612Running coprocessor pre-close hooks at 1732301813612Disabling compacts and flushes for region at 1732301813612Disabling writes for close at 1732301814693 (+1081 ms)Obtaining lock to block concurrent updates at 1732301814693Preparing flush snapshotting stores in 2f5686bd89d51ddd0842891a5a86fd98 at 1732301814693Finished memstore snapshotting TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98., syncing WAL and waiting on mvcc, flushsize=dataSize=6870, getHeapSize=18720, getOffHeapSize=0, getCellsCount=150 at 1732301814694 (+1 ms)Flushing stores of TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98. at 1732301814694Flushing 2f5686bd89d51ddd0842891a5a86fd98/A: creating writer at 1732301814694Flushing 2f5686bd89d51ddd0842891a5a86fd98/A: appending metadata at 1732301814696 (+2 ms)Flushing 2f5686bd89d51ddd0842891a5a86fd98/A: closing flushed file at 1732301814696Flushing 2f5686bd89d51ddd0842891a5a86fd98/B: creating writer at 1732301815104 (+408 ms)Flushing 2f5686bd89d51ddd0842891a5a86fd98/B: appending metadata at 1732301815106 (+2 ms)Flushing 2f5686bd89d51ddd0842891a5a86fd98/B: closing flushed file at 1732301815106Flushing 2f5686bd89d51ddd0842891a5a86fd98/C: creating writer at 1732301815514 (+408 ms)Flushing 2f5686bd89d51ddd0842891a5a86fd98/C: appending metadata at 1732301815517 (+3 ms)Flushing 2f5686bd89d51ddd0842891a5a86fd98/C: closing flushed file at 1732301815517Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@dbbbdca: reopening flushed file at 1732301815924 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@416c21af: reopening flushed file at 1732301815928 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@460d6405: reopening flushed file at 1732301815931 (+3 ms)Finished flush of dataSize ~6.71 KB/6870, heapSize ~18.28 KB/18720, currentSize=0 B/0 for 2f5686bd89d51ddd0842891a5a86fd98 in 1242ms, sequenceid=465, compaction requested=true at 1732301815936 (+5 ms)Writing region close event to WAL at 1732301816030 (+94 ms)Running coprocessor post-close hooks at 1732301816033 (+3 ms)Closed at 1732301816033 2024-11-22T18:56:56,035 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(157): Closed 2f5686bd89d51ddd0842891a5a86fd98 2024-11-22T18:56:56,035 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=2f5686bd89d51ddd0842891a5a86fd98, regionState=CLOSED 2024-11-22T18:56:56,036 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=97, ppid=96, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 because future has completed 2024-11-22T18:56:56,038 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=97, resume processing ppid=96 2024-11-22T18:56:56,038 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, ppid=96, state=SUCCESS, hasLock=false; CloseRegionProcedure 2f5686bd89d51ddd0842891a5a86fd98, server=d79ba0c344fb,43593,1732301724173 in 2.5770 sec 2024-11-22T18:56:56,040 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-22T18:56:56,040 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2f5686bd89d51ddd0842891a5a86fd98, UNASSIGN in 2.5810 sec 2024-11-22T18:56:56,042 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-11-22T18:56:56,042 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.5840 sec 2024-11-22T18:56:56,043 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732301816043"}]},"ts":"1732301816043"} 2024-11-22T18:56:56,044 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-22T18:56:56,044 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-22T18:56:56,046 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.5930 sec 2024-11-22T18:56:57,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-22T18:56:57,616 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-22T18:56:57,617 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-22T18:56:57,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] procedure2.ProcedureExecutor(1139): Stored pid=98, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T18:56:57,619 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=98, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T18:56:57,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=98 2024-11-22T18:56:57,620 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=98, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T18:56:57,622 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98 2024-11-22T18:56:57,624 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A, FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B, FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C, FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/recovered.edits] 2024-11-22T18:56:57,627 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/148eab2460334f9ba4cceaefd7679ab2 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/148eab2460334f9ba4cceaefd7679ab2 2024-11-22T18:56:57,629 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/333190869edb4634bcbb32f8ab0f49be to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/333190869edb4634bcbb32f8ab0f49be 2024-11-22T18:56:57,630 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/4a198b552cb8438eafc2c9aab888d5f5 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/A/4a198b552cb8438eafc2c9aab888d5f5 2024-11-22T18:56:57,633 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/7da54b79c0fa4624bfa98babc055eec4 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/7da54b79c0fa4624bfa98babc055eec4 2024-11-22T18:56:57,634 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/80a57c014e6c43c79bdc0d2df9e9af13 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/80a57c014e6c43c79bdc0d2df9e9af13 2024-11-22T18:56:57,635 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/cc2a0ee238d44250995857503366fabd to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/B/cc2a0ee238d44250995857503366fabd 2024-11-22T18:56:57,637 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/34d8b7aaca054ca5b4cd547346e5c426 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/34d8b7aaca054ca5b4cd547346e5c426 2024-11-22T18:56:57,638 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/36c4b16a26ae427eb12b6d47a371aa53 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/36c4b16a26ae427eb12b6d47a371aa53 2024-11-22T18:56:57,639 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/84a36a1491034585a91ea05bf3c5f36d to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/C/84a36a1491034585a91ea05bf3c5f36d 2024-11-22T18:56:57,641 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/recovered.edits/468.seqid to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98/recovered.edits/468.seqid 2024-11-22T18:56:57,641 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2f5686bd89d51ddd0842891a5a86fd98 2024-11-22T18:56:57,642 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-22T18:56:57,643 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=98, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T18:56:57,646 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-22T18:56:57,648 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-22T18:56:57,649 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=98, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T18:56:57,649 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-22T18:56:57,649 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732301817649"}]},"ts":"9223372036854775807"} 2024-11-22T18:56:57,650 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-22T18:56:57,650 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 2f5686bd89d51ddd0842891a5a86fd98, NAME => 'TestAcidGuarantees,,1732301789364.2f5686bd89d51ddd0842891a5a86fd98.', STARTKEY => '', ENDKEY => ''}] 2024-11-22T18:56:57,650 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-22T18:56:57,651 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732301817650"}]},"ts":"9223372036854775807"} 2024-11-22T18:56:57,652 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-22T18:56:57,653 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=98, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T18:56:57,654 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 36 msec 2024-11-22T18:56:57,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=98 2024-11-22T18:56:57,726 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-22T18:56:57,726 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-22T18:56:57,735 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=240 (was 240), OpenFileDescriptor=455 (was 451) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=304 (was 317), ProcessCount=11 (was 11), AvailableMemoryMB=7809 (was 7945) 2024-11-22T18:56:57,743 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=240, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=304, ProcessCount=11, AvailableMemoryMB=7808 2024-11-22T18:56:57,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-22T18:56:57,745 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T18:56:57,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-22T18:56:57,747 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=99, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-22T18:56:57,747 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:56:57,747 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 99 2024-11-22T18:56:57,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-22T18:56:57,747 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=99, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-22T18:56:57,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742141_1317 (size=959) 2024-11-22T18:56:57,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-22T18:56:58,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-22T18:56:58,154 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 2c052b4ec10710079fa7898a974e92e1, NAME => 'TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875 2024-11-22T18:56:58,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742142_1318 (size=53) 2024-11-22T18:56:58,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-22T18:56:58,559 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T18:56:58,559 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 2c052b4ec10710079fa7898a974e92e1, disabling compactions & flushes 2024-11-22T18:56:58,559 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:56:58,559 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:56:58,559 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. after waiting 0 ms 2024-11-22T18:56:58,559 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:56:58,559 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:56:58,560 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 2c052b4ec10710079fa7898a974e92e1: Waiting for close lock at 1732301818559Disabling compacts and flushes for region at 1732301818559Disabling writes for close at 1732301818559Writing region close event to WAL at 1732301818559Closed at 1732301818559 2024-11-22T18:56:58,561 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=99, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-22T18:56:58,561 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732301818561"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732301818561"}]},"ts":"1732301818561"} 2024-11-22T18:56:58,562 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-22T18:56:58,563 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=99, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-22T18:56:58,563 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732301818563"}]},"ts":"1732301818563"} 2024-11-22T18:56:58,564 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-22T18:56:58,564 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2c052b4ec10710079fa7898a974e92e1, ASSIGN}] 2024-11-22T18:56:58,565 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=100, ppid=99, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2c052b4ec10710079fa7898a974e92e1, ASSIGN 2024-11-22T18:56:58,566 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=100, ppid=99, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2c052b4ec10710079fa7898a974e92e1, ASSIGN; state=OFFLINE, location=d79ba0c344fb,43593,1732301724173; forceNewPlan=false, retain=false 2024-11-22T18:56:58,716 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=100 updating hbase:meta row=2c052b4ec10710079fa7898a974e92e1, regionState=OPENING, regionLocation=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:58,718 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=100, ppid=99, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2c052b4ec10710079fa7898a974e92e1, ASSIGN because future has completed 2024-11-22T18:56:58,718 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=101, ppid=100, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173}] 2024-11-22T18:56:58,873 INFO [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:56:58,873 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(7752): Opening region: {ENCODED => 2c052b4ec10710079fa7898a974e92e1, NAME => 'TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.', STARTKEY => '', ENDKEY => ''} 2024-11-22T18:56:58,873 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 2c052b4ec10710079fa7898a974e92e1 2024-11-22T18:56:58,873 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T18:56:58,873 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(7794): checking encryption for 2c052b4ec10710079fa7898a974e92e1 2024-11-22T18:56:58,873 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(7797): checking classloading for 2c052b4ec10710079fa7898a974e92e1 2024-11-22T18:56:58,874 INFO [StoreOpener-2c052b4ec10710079fa7898a974e92e1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 2c052b4ec10710079fa7898a974e92e1 2024-11-22T18:56:58,875 INFO [StoreOpener-2c052b4ec10710079fa7898a974e92e1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2c052b4ec10710079fa7898a974e92e1 columnFamilyName A 2024-11-22T18:56:58,876 DEBUG [StoreOpener-2c052b4ec10710079fa7898a974e92e1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:56:58,876 INFO [StoreOpener-2c052b4ec10710079fa7898a974e92e1-1 {}] regionserver.HStore(327): Store=2c052b4ec10710079fa7898a974e92e1/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T18:56:58,876 INFO [StoreOpener-2c052b4ec10710079fa7898a974e92e1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 2c052b4ec10710079fa7898a974e92e1 2024-11-22T18:56:58,877 INFO [StoreOpener-2c052b4ec10710079fa7898a974e92e1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2c052b4ec10710079fa7898a974e92e1 columnFamilyName B 2024-11-22T18:56:58,877 DEBUG [StoreOpener-2c052b4ec10710079fa7898a974e92e1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:56:58,877 INFO [StoreOpener-2c052b4ec10710079fa7898a974e92e1-1 {}] regionserver.HStore(327): Store=2c052b4ec10710079fa7898a974e92e1/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T18:56:58,877 INFO [StoreOpener-2c052b4ec10710079fa7898a974e92e1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 2c052b4ec10710079fa7898a974e92e1 2024-11-22T18:56:58,878 INFO [StoreOpener-2c052b4ec10710079fa7898a974e92e1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2c052b4ec10710079fa7898a974e92e1 columnFamilyName C 2024-11-22T18:56:58,878 DEBUG [StoreOpener-2c052b4ec10710079fa7898a974e92e1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:56:58,878 INFO [StoreOpener-2c052b4ec10710079fa7898a974e92e1-1 {}] regionserver.HStore(327): Store=2c052b4ec10710079fa7898a974e92e1/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T18:56:58,879 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(1038): replaying wal for 2c052b4ec10710079fa7898a974e92e1 2024-11-22T18:56:58,879 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1 2024-11-22T18:56:58,879 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1 2024-11-22T18:56:58,880 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(1048): stopping wal replay for 2c052b4ec10710079fa7898a974e92e1 2024-11-22T18:56:58,880 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(1060): Cleaning up temporary data for 2c052b4ec10710079fa7898a974e92e1 2024-11-22T18:56:58,881 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-22T18:56:58,881 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(1093): writing seq id for 2c052b4ec10710079fa7898a974e92e1 2024-11-22T18:56:58,883 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T18:56:58,883 INFO [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(1114): Opened 2c052b4ec10710079fa7898a974e92e1; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63516599, jitterRate=-0.05352891981601715}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-22T18:56:58,883 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2c052b4ec10710079fa7898a974e92e1 2024-11-22T18:56:58,884 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(1006): Region open journal for 2c052b4ec10710079fa7898a974e92e1: Running coprocessor pre-open hook at 1732301818873Writing region info on filesystem at 1732301818873Initializing all the Stores at 1732301818874 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732301818874Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732301818874Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732301818874Cleaning up temporary data from old regions at 1732301818880 (+6 ms)Running coprocessor post-open hooks at 1732301818883 (+3 ms)Region opened successfully at 1732301818884 (+1 ms) 2024-11-22T18:56:58,884 INFO [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., pid=101, masterSystemTime=1732301818870 2024-11-22T18:56:58,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-22T18:56:58,886 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:56:58,886 INFO [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:56:58,886 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=100 updating hbase:meta row=2c052b4ec10710079fa7898a974e92e1, regionState=OPEN, openSeqNum=2, regionLocation=d79ba0c344fb,43593,1732301724173 2024-11-22T18:56:58,888 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=101, ppid=100, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 because future has completed 2024-11-22T18:56:58,888 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44109 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=d79ba0c344fb,43593,1732301724173, table=TestAcidGuarantees, region=2c052b4ec10710079fa7898a974e92e1. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-22T18:56:58,890 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=101, resume processing ppid=100 2024-11-22T18:56:58,890 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, ppid=100, state=SUCCESS, hasLock=false; OpenRegionProcedure 2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 in 170 msec 2024-11-22T18:56:58,891 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-22T18:56:58,891 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2c052b4ec10710079fa7898a974e92e1, ASSIGN in 326 msec 2024-11-22T18:56:58,892 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=99, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-22T18:56:58,892 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732301818892"}]},"ts":"1732301818892"} 2024-11-22T18:56:58,893 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-22T18:56:58,894 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=99, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-22T18:56:58,895 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1490 sec 2024-11-22T18:56:59,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-22T18:56:59,896 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-22T18:56:59,897 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29f9102f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:56:59,897 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d79ba0c344fb,44109,-1 for getting cluster id 2024-11-22T18:56:59,897 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T18:56:59,898 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '058c0dbe-7061-419a-804d-605a1c1b6341' 2024-11-22T18:56:59,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T18:56:59,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "058c0dbe-7061-419a-804d-605a1c1b6341" 2024-11-22T18:56:59,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a921a35, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:56:59,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d79ba0c344fb,44109,-1] 2024-11-22T18:56:59,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T18:56:59,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:56:59,900 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44518, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T18:56:59,900 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@317c61d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:56:59,901 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T18:56:59,901 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d79ba0c344fb,43593,1732301724173, seqNum=-1] 2024-11-22T18:56:59,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T18:56:59,902 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37326, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T18:56:59,903 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-22T18:56:59,904 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is d79ba0c344fb,44109,1732301723398 2024-11-22T18:56:59,904 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@70ec5a68 2024-11-22T18:56:59,904 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-22T18:56:59,905 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44530, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-22T18:56:59,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-22T18:56:59,906 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T18:56:59,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] procedure2.ProcedureExecutor(1139): Stored pid=102, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-22T18:56:59,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742143_1319 (size=995) 2024-11-22T18:57:00,317 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-22T18:57:00,317 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-22T18:57:00,319 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=103, ppid=102, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-22T18:57:00,321 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2c052b4ec10710079fa7898a974e92e1, REOPEN/MOVE}] 2024-11-22T18:57:00,321 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=104, ppid=103, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2c052b4ec10710079fa7898a974e92e1, REOPEN/MOVE 2024-11-22T18:57:00,322 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=104 updating hbase:meta row=2c052b4ec10710079fa7898a974e92e1, regionState=CLOSING, regionLocation=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:00,323 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=104, ppid=103, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2c052b4ec10710079fa7898a974e92e1, REOPEN/MOVE because future has completed 2024-11-22T18:57:00,323 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-22T18:57:00,323 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=105, ppid=104, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173}] 2024-11-22T18:57:00,475 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] handler.UnassignRegionHandler(122): Close 2c052b4ec10710079fa7898a974e92e1 2024-11-22T18:57:00,475 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-22T18:57:00,475 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] regionserver.HRegion(1722): Closing 2c052b4ec10710079fa7898a974e92e1, disabling compactions & flushes 2024-11-22T18:57:00,475 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:57:00,475 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:57:00,476 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. after waiting 0 ms 2024-11-22T18:57:00,476 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:57:00,479 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-22T18:57:00,479 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:57:00,479 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] regionserver.HRegion(1676): Region close journal for 2c052b4ec10710079fa7898a974e92e1: Waiting for close lock at 1732301820475Running coprocessor pre-close hooks at 1732301820475Disabling compacts and flushes for region at 1732301820475Disabling writes for close at 1732301820476 (+1 ms)Writing region close event to WAL at 1732301820476Running coprocessor post-close hooks at 1732301820479 (+3 ms)Closed at 1732301820479 2024-11-22T18:57:00,480 WARN [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] regionserver.HRegionServer(3299): Not adding moved region record: 2c052b4ec10710079fa7898a974e92e1 to self. 2024-11-22T18:57:00,481 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] handler.UnassignRegionHandler(157): Closed 2c052b4ec10710079fa7898a974e92e1 2024-11-22T18:57:00,481 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=104 updating hbase:meta row=2c052b4ec10710079fa7898a974e92e1, regionState=CLOSED 2024-11-22T18:57:00,483 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=105, ppid=104, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 because future has completed 2024-11-22T18:57:00,484 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=105, resume processing ppid=104 2024-11-22T18:57:00,484 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, ppid=104, state=SUCCESS, hasLock=false; CloseRegionProcedure 2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 in 160 msec 2024-11-22T18:57:00,485 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=104, ppid=103, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2c052b4ec10710079fa7898a974e92e1, REOPEN/MOVE; state=CLOSED, location=d79ba0c344fb,43593,1732301724173; forceNewPlan=false, retain=true 2024-11-22T18:57:00,636 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=104 updating hbase:meta row=2c052b4ec10710079fa7898a974e92e1, regionState=OPENING, regionLocation=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:00,637 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=104, ppid=103, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2c052b4ec10710079fa7898a974e92e1, REOPEN/MOVE because future has completed 2024-11-22T18:57:00,637 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=104, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173}] 2024-11-22T18:57:00,791 INFO [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:57:00,791 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(7752): Opening region: {ENCODED => 2c052b4ec10710079fa7898a974e92e1, NAME => 'TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.', STARTKEY => '', ENDKEY => ''} 2024-11-22T18:57:00,792 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 2c052b4ec10710079fa7898a974e92e1 2024-11-22T18:57:00,792 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T18:57:00,792 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(7794): checking encryption for 2c052b4ec10710079fa7898a974e92e1 2024-11-22T18:57:00,792 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(7797): checking classloading for 2c052b4ec10710079fa7898a974e92e1 2024-11-22T18:57:00,793 INFO [StoreOpener-2c052b4ec10710079fa7898a974e92e1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 2c052b4ec10710079fa7898a974e92e1 2024-11-22T18:57:00,793 INFO [StoreOpener-2c052b4ec10710079fa7898a974e92e1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2c052b4ec10710079fa7898a974e92e1 columnFamilyName A 2024-11-22T18:57:00,794 DEBUG [StoreOpener-2c052b4ec10710079fa7898a974e92e1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:57:00,795 INFO [StoreOpener-2c052b4ec10710079fa7898a974e92e1-1 {}] regionserver.HStore(327): Store=2c052b4ec10710079fa7898a974e92e1/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T18:57:00,795 INFO [StoreOpener-2c052b4ec10710079fa7898a974e92e1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 2c052b4ec10710079fa7898a974e92e1 2024-11-22T18:57:00,796 INFO [StoreOpener-2c052b4ec10710079fa7898a974e92e1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2c052b4ec10710079fa7898a974e92e1 columnFamilyName B 2024-11-22T18:57:00,796 DEBUG [StoreOpener-2c052b4ec10710079fa7898a974e92e1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:57:00,796 INFO [StoreOpener-2c052b4ec10710079fa7898a974e92e1-1 {}] regionserver.HStore(327): Store=2c052b4ec10710079fa7898a974e92e1/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T18:57:00,796 INFO [StoreOpener-2c052b4ec10710079fa7898a974e92e1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 2c052b4ec10710079fa7898a974e92e1 2024-11-22T18:57:00,797 INFO [StoreOpener-2c052b4ec10710079fa7898a974e92e1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2c052b4ec10710079fa7898a974e92e1 columnFamilyName C 2024-11-22T18:57:00,797 DEBUG [StoreOpener-2c052b4ec10710079fa7898a974e92e1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:57:00,797 INFO [StoreOpener-2c052b4ec10710079fa7898a974e92e1-1 {}] regionserver.HStore(327): Store=2c052b4ec10710079fa7898a974e92e1/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T18:57:00,797 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1038): replaying wal for 2c052b4ec10710079fa7898a974e92e1 2024-11-22T18:57:00,798 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1 2024-11-22T18:57:00,799 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1 2024-11-22T18:57:00,799 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1048): stopping wal replay for 2c052b4ec10710079fa7898a974e92e1 2024-11-22T18:57:00,799 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1060): Cleaning up temporary data for 2c052b4ec10710079fa7898a974e92e1 2024-11-22T18:57:00,800 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-22T18:57:00,800 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1093): writing seq id for 2c052b4ec10710079fa7898a974e92e1 2024-11-22T18:57:00,801 INFO [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1114): Opened 2c052b4ec10710079fa7898a974e92e1; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65024688, jitterRate=-0.031056642532348633}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-22T18:57:00,801 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2c052b4ec10710079fa7898a974e92e1 2024-11-22T18:57:00,802 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1006): Region open journal for 2c052b4ec10710079fa7898a974e92e1: Running coprocessor pre-open hook at 1732301820792Writing region info on filesystem at 1732301820792Initializing all the Stores at 1732301820792Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732301820792Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732301820792Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732301820793 (+1 ms)Cleaning up temporary data from old regions at 1732301820799 (+6 ms)Running coprocessor post-open hooks at 1732301820801 (+2 ms)Region opened successfully at 1732301820802 (+1 ms) 2024-11-22T18:57:00,802 INFO [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., pid=106, masterSystemTime=1732301820788 2024-11-22T18:57:00,804 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:57:00,804 INFO [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:57:00,804 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=104 updating hbase:meta row=2c052b4ec10710079fa7898a974e92e1, regionState=OPEN, openSeqNum=5, regionLocation=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:00,805 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=106, ppid=104, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 because future has completed 2024-11-22T18:57:00,807 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=104 2024-11-22T18:57:00,807 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=104, state=SUCCESS, hasLock=false; OpenRegionProcedure 2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 in 169 msec 2024-11-22T18:57:00,809 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-22T18:57:00,809 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2c052b4ec10710079fa7898a974e92e1, REOPEN/MOVE in 486 msec 2024-11-22T18:57:00,811 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=103, resume processing ppid=102 2024-11-22T18:57:00,811 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, ppid=102, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 490 msec 2024-11-22T18:57:00,813 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 906 msec 2024-11-22T18:57:00,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=102 2024-11-22T18:57:00,814 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-22T18:57:00,814 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ec1958f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:57:00,814 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d79ba0c344fb,44109,-1 for getting cluster id 2024-11-22T18:57:00,814 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T18:57:00,815 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '058c0dbe-7061-419a-804d-605a1c1b6341' 2024-11-22T18:57:00,815 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T18:57:00,815 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "058c0dbe-7061-419a-804d-605a1c1b6341" 2024-11-22T18:57:00,815 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42323a80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:57:00,815 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d79ba0c344fb,44109,-1] 2024-11-22T18:57:00,816 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T18:57:00,816 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:57:00,816 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46108, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T18:57:00,817 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31e66f73, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:57:00,817 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b2f185f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:57:00,817 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d79ba0c344fb,44109,-1 for getting cluster id 2024-11-22T18:57:00,817 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T18:57:00,818 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '058c0dbe-7061-419a-804d-605a1c1b6341' 2024-11-22T18:57:00,818 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T18:57:00,818 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "058c0dbe-7061-419a-804d-605a1c1b6341" 2024-11-22T18:57:00,818 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d3238e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:57:00,818 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d79ba0c344fb,44109,-1] 2024-11-22T18:57:00,818 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T18:57:00,819 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:57:00,819 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46122, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T18:57:00,819 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ac523fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:57:00,820 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a739f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:57:00,820 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d79ba0c344fb,44109,-1 for getting cluster id 2024-11-22T18:57:00,820 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T18:57:00,821 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '058c0dbe-7061-419a-804d-605a1c1b6341' 2024-11-22T18:57:00,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T18:57:00,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "058c0dbe-7061-419a-804d-605a1c1b6341" 2024-11-22T18:57:00,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bcd0b55, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:57:00,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d79ba0c344fb,44109,-1] 2024-11-22T18:57:00,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T18:57:00,822 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:57:00,822 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46152, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T18:57:00,822 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cbe458c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:57:00,823 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@87fd48f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:57:00,823 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d79ba0c344fb,44109,-1 for getting cluster id 2024-11-22T18:57:00,823 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T18:57:00,823 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '058c0dbe-7061-419a-804d-605a1c1b6341' 2024-11-22T18:57:00,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T18:57:00,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "058c0dbe-7061-419a-804d-605a1c1b6341" 2024-11-22T18:57:00,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@260d7dcc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:57:00,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d79ba0c344fb,44109,-1] 2024-11-22T18:57:00,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T18:57:00,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:57:00,825 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46158, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T18:57:00,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25b83de6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:57:00,826 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f0bd6a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:57:00,826 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d79ba0c344fb,44109,-1 for getting cluster id 2024-11-22T18:57:00,826 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T18:57:00,826 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '058c0dbe-7061-419a-804d-605a1c1b6341' 2024-11-22T18:57:00,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T18:57:00,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "058c0dbe-7061-419a-804d-605a1c1b6341" 2024-11-22T18:57:00,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f155679, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:57:00,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d79ba0c344fb,44109,-1] 2024-11-22T18:57:00,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T18:57:00,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:57:00,828 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46184, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T18:57:00,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41b2693d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:57:00,828 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dd3a90f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:57:00,828 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d79ba0c344fb,44109,-1 for getting cluster id 2024-11-22T18:57:00,829 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T18:57:00,829 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '058c0dbe-7061-419a-804d-605a1c1b6341' 2024-11-22T18:57:00,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T18:57:00,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "058c0dbe-7061-419a-804d-605a1c1b6341" 2024-11-22T18:57:00,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d7c4bdc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:57:00,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d79ba0c344fb,44109,-1] 2024-11-22T18:57:00,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T18:57:00,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:57:00,830 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46202, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T18:57:00,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44e5158b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:57:00,831 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10e2b5a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:57:00,831 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d79ba0c344fb,44109,-1 for getting cluster id 2024-11-22T18:57:00,831 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T18:57:00,832 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '058c0dbe-7061-419a-804d-605a1c1b6341' 2024-11-22T18:57:00,832 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T18:57:00,832 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "058c0dbe-7061-419a-804d-605a1c1b6341" 2024-11-22T18:57:00,832 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e66ed2f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:57:00,832 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d79ba0c344fb,44109,-1] 2024-11-22T18:57:00,832 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T18:57:00,832 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:57:00,833 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46226, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T18:57:00,833 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33a4f16f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:57:00,834 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d41a701, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:57:00,834 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d79ba0c344fb,44109,-1 for getting cluster id 2024-11-22T18:57:00,834 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T18:57:00,834 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '058c0dbe-7061-419a-804d-605a1c1b6341' 2024-11-22T18:57:00,834 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T18:57:00,835 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "058c0dbe-7061-419a-804d-605a1c1b6341" 2024-11-22T18:57:00,835 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75d86f93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:57:00,835 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d79ba0c344fb,44109,-1] 2024-11-22T18:57:00,835 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T18:57:00,835 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:57:00,835 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46232, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T18:57:00,836 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bbc6202, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:57:00,836 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b2f46ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:57:00,836 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d79ba0c344fb,44109,-1 for getting cluster id 2024-11-22T18:57:00,837 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T18:57:00,837 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '058c0dbe-7061-419a-804d-605a1c1b6341' 2024-11-22T18:57:00,837 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T18:57:00,837 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "058c0dbe-7061-419a-804d-605a1c1b6341" 2024-11-22T18:57:00,837 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4578091e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:57:00,837 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d79ba0c344fb,44109,-1] 2024-11-22T18:57:00,838 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T18:57:00,838 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:57:00,838 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46256, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T18:57:00,839 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64bf9aef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:57:00,839 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@180bd544, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:57:00,839 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d79ba0c344fb,44109,-1 for getting cluster id 2024-11-22T18:57:00,839 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T18:57:00,840 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '058c0dbe-7061-419a-804d-605a1c1b6341' 2024-11-22T18:57:00,840 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T18:57:00,840 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "058c0dbe-7061-419a-804d-605a1c1b6341" 2024-11-22T18:57:00,840 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4849a664, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:57:00,840 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d79ba0c344fb,44109,-1] 2024-11-22T18:57:00,840 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T18:57:00,840 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:57:00,841 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46274, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T18:57:00,841 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b1dcee7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:57:00,842 DEBUG [Thread-1464 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T18:57:00,842 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T18:57:00,842 DEBUG [Thread-1477 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T18:57:00,842 DEBUG [Thread-1471 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T18:57:00,842 DEBUG [Thread-1479 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T18:57:00,843 DEBUG [Thread-1475 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T18:57:00,843 DEBUG [Thread-1473 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T18:57:00,843 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d79ba0c344fb,43593,1732301724173, seqNum=-1] 2024-11-22T18:57:00,843 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d79ba0c344fb,43593,1732301724173, seqNum=-1] 2024-11-22T18:57:00,843 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d79ba0c344fb,43593,1732301724173, seqNum=-1] 2024-11-22T18:57:00,843 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T18:57:00,843 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T18:57:00,843 DEBUG [Thread-1468 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T18:57:00,844 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T18:57:00,844 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d79ba0c344fb,43593,1732301724173, seqNum=-1] 2024-11-22T18:57:00,844 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d79ba0c344fb,43593,1732301724173, seqNum=-1] 2024-11-22T18:57:00,844 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T18:57:00,844 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d79ba0c344fb,43593,1732301724173, seqNum=-1] 2024-11-22T18:57:00,844 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T18:57:00,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-22T18:57:00,844 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T18:57:00,844 DEBUG [Thread-1462 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T18:57:00,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-22T18:57:00,845 DEBUG [Thread-1460 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T18:57:00,845 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41666, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T18:57:00,845 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41680, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T18:57:00,846 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41668, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T18:57:00,846 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41700, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T18:57:00,846 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41692, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T18:57:00,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d79ba0c344fb,43593,1732301724173, seqNum=-1] 2024-11-22T18:57:00,846 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d79ba0c344fb,43593,1732301724173, seqNum=-1] 2024-11-22T18:57:00,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T18:57:00,846 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41694, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T18:57:00,846 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T18:57:00,847 DEBUG [Thread-1466 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T18:57:00,847 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d79ba0c344fb,43593,1732301724173, seqNum=-1] 2024-11-22T18:57:00,847 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T18:57:00,847 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41708, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T18:57:00,847 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41718, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T18:57:00,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d79ba0c344fb,43593,1732301724173, seqNum=-1] 2024-11-22T18:57:00,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T18:57:00,848 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T18:57:00,848 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41730, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T18:57:00,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5] 2024-11-22T18:57:00,849 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5] 2024-11-22T18:57:00,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5] 2024-11-22T18:57:00,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5] 2024-11-22T18:57:00,851 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5] 2024-11-22T18:57:00,851 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5] 2024-11-22T18:57:00,851 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41726, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T18:57:00,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5] 2024-11-22T18:57:00,852 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5] 2024-11-22T18:57:00,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5] 2024-11-22T18:57:00,853 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T18:57:00,853 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T18:57:00,853 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5] 2024-11-22T18:57:00,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on 2c052b4ec10710079fa7898a974e92e1 2024-11-22T18:57:00,859 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T18:57:00,859 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2c052b4ec10710079fa7898a974e92e1 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-22T18:57:00,889 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122d7c5186dd0ea430c994d506caeb34115_2c052b4ec10710079fa7898a974e92e1 is 50, key is test_row_0/A:col10/1732301820853/Put/seqid=0 2024-11-22T18:57:00,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742144_1320 (size=14594) 2024-11-22T18:57:00,902 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:57:00,906 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122d7c5186dd0ea430c994d506caeb34115_2c052b4ec10710079fa7898a974e92e1 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122d7c5186dd0ea430c994d506caeb34115_2c052b4ec10710079fa7898a974e92e1 2024-11-22T18:57:00,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:00,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41726 deadline: 1732301880895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:00,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:00,906 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:00,906 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:00,906 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:00,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41730 deadline: 1732301880901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:00,906 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/A/074178ce8a1b41f5abcc83b728726f37, store: [table=TestAcidGuarantees family=A region=2c052b4ec10710079fa7898a974e92e1] 2024-11-22T18:57:00,907 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:00,907 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:00,907 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:00,907 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/A/074178ce8a1b41f5abcc83b728726f37 is 175, key is test_row_0/A:col10/1732301820853/Put/seqid=0 2024-11-22T18:57:00,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742145_1321 (size=39549) 2024-11-22T18:57:00,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:00,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732301880902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:00,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:00,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:00,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732301880906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:00,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:00,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:00,913 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:00,913 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:00,913 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:00,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:00,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732301880907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:00,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:00,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:00,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:00,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-22T18:57:01,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-22T18:57:01,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:57:01,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. as already flushing 2024-11-22T18:57:01,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:57:01,006 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 2c052b4ec10710079fa7898a974e92e1, NAME => 'TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:01,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 2c052b4ec10710079fa7898a974e92e1, NAME => 'TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:01,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2c052b4ec10710079fa7898a974e92e1, NAME => 'TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2c052b4ec10710079fa7898a974e92e1, NAME => 'TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:01,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:01,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41726 deadline: 1732301881016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:01,022 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:01,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:01,022 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:01,022 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:01,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41730 deadline: 1732301881016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:01,023 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:01,023 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:01,023 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:01,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:01,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732301881016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:01,023 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:01,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:01,023 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:01,023 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:01,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732301881016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:01,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:01,024 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:01,024 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:01,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:01,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732301881025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:01,035 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:01,035 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:01,035 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:01,159 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-22T18:57:01,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:57:01,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. as already flushing 2024-11-22T18:57:01,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:57:01,159 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 2c052b4ec10710079fa7898a974e92e1, NAME => 'TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:01,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 2c052b4ec10710079fa7898a974e92e1, NAME => 'TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:01,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2c052b4ec10710079fa7898a974e92e1, NAME => 'TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2c052b4ec10710079fa7898a974e92e1, NAME => 'TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:01,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-22T18:57:01,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:01,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41726 deadline: 1732301881226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:01,232 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:01,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:01,232 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:01,232 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:01,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41730 deadline: 1732301881226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:01,233 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:01,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:01,233 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:01,233 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:01,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732301881226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:01,233 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:01,233 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:01,233 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:01,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:01,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732301881235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:01,243 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:01,243 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:01,243 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:01,254 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:01,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732301881245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:01,254 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:01,254 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:01,254 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:01,311 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/A/074178ce8a1b41f5abcc83b728726f37 2024-11-22T18:57:01,315 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-22T18:57:01,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:57:01,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. as already flushing 2024-11-22T18:57:01,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:57:01,315 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 2c052b4ec10710079fa7898a974e92e1, NAME => 'TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:01,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 2c052b4ec10710079fa7898a974e92e1, NAME => 'TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:01,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2c052b4ec10710079fa7898a974e92e1, NAME => 'TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2c052b4ec10710079fa7898a974e92e1, NAME => 'TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:01,337 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/B/5fbb26f6a2a8440789922d1f4fd3694a is 50, key is test_row_0/B:col10/1732301820853/Put/seqid=0 2024-11-22T18:57:01,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742146_1322 (size=12001) 2024-11-22T18:57:01,469 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-22T18:57:01,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:57:01,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. as already flushing 2024-11-22T18:57:01,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:57:01,469 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 2c052b4ec10710079fa7898a974e92e1, NAME => 'TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:01,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 2c052b4ec10710079fa7898a974e92e1, NAME => 'TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:01,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2c052b4ec10710079fa7898a974e92e1, NAME => 'TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2c052b4ec10710079fa7898a974e92e1, NAME => 'TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:01,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-22T18:57:01,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:01,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41726 deadline: 1732301881536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:01,540 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:01,540 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:01,540 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:01,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:01,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41730 deadline: 1732301881546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:01,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:01,551 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:01,551 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:01,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732301881546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:01,551 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:01,552 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:01,552 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:01,552 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:01,556 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-22T18:57:01,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:01,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732301881555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:01,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:01,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:01,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:01,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:01,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732301881565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:01,574 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:01,574 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:01,574 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:01,622 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-22T18:57:01,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:57:01,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. as already flushing 2024-11-22T18:57:01,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:57:01,622 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 2c052b4ec10710079fa7898a974e92e1, NAME => 'TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:01,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 2c052b4ec10710079fa7898a974e92e1, NAME => 'TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:01,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2c052b4ec10710079fa7898a974e92e1, NAME => 'TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2c052b4ec10710079fa7898a974e92e1, NAME => 'TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:01,742 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/B/5fbb26f6a2a8440789922d1f4fd3694a 2024-11-22T18:57:01,767 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/C/a85f58865dbb407bb7a45313653fd2d1 is 50, key is test_row_0/C:col10/1732301820853/Put/seqid=0 2024-11-22T18:57:01,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742147_1323 (size=12001) 2024-11-22T18:57:01,775 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-22T18:57:01,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:57:01,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. as already flushing 2024-11-22T18:57:01,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:57:01,775 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 2c052b4ec10710079fa7898a974e92e1, NAME => 'TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:01,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 2c052b4ec10710079fa7898a974e92e1, NAME => 'TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:01,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2c052b4ec10710079fa7898a974e92e1, NAME => 'TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2c052b4ec10710079fa7898a974e92e1, NAME => 'TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:01,928 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-22T18:57:01,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:57:01,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. as already flushing 2024-11-22T18:57:01,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:57:01,928 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 2c052b4ec10710079fa7898a974e92e1, NAME => 'TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:01,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 2c052b4ec10710079fa7898a974e92e1, NAME => 'TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:01,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2c052b4ec10710079fa7898a974e92e1, NAME => 'TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2c052b4ec10710079fa7898a974e92e1, NAME => 'TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:01,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-22T18:57:02,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:02,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41726 deadline: 1732301882045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:02,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:02,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:02,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:02,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:02,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41730 deadline: 1732301882055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:02,065 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:02,065 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:02,065 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:02,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:02,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732301882065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:02,066 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:02,066 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:02,066 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:02,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:02,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732301882076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:02,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:02,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:02,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:02,081 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-22T18:57:02,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:57:02,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. as already flushing 2024-11-22T18:57:02,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:57:02,082 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 2c052b4ec10710079fa7898a974e92e1, NAME => 'TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:02,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 2c052b4ec10710079fa7898a974e92e1, NAME => 'TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:02,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2c052b4ec10710079fa7898a974e92e1, NAME => 'TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2c052b4ec10710079fa7898a974e92e1, NAME => 'TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:02,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:02,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732301882086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:02,089 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:02,089 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:02,089 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:02,171 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/C/a85f58865dbb407bb7a45313653fd2d1 2024-11-22T18:57:02,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/A/074178ce8a1b41f5abcc83b728726f37 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/074178ce8a1b41f5abcc83b728726f37 2024-11-22T18:57:02,179 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/074178ce8a1b41f5abcc83b728726f37, entries=200, sequenceid=16, filesize=38.6 K 2024-11-22T18:57:02,180 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/B/5fbb26f6a2a8440789922d1f4fd3694a as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/5fbb26f6a2a8440789922d1f4fd3694a 2024-11-22T18:57:02,184 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/5fbb26f6a2a8440789922d1f4fd3694a, entries=150, sequenceid=16, filesize=11.7 K 2024-11-22T18:57:02,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/C/a85f58865dbb407bb7a45313653fd2d1 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/a85f58865dbb407bb7a45313653fd2d1 2024-11-22T18:57:02,188 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/a85f58865dbb407bb7a45313653fd2d1, entries=150, sequenceid=16, filesize=11.7 K 2024-11-22T18:57:02,189 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 2c052b4ec10710079fa7898a974e92e1 in 1329ms, sequenceid=16, compaction requested=false 2024-11-22T18:57:02,189 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-22T18:57:02,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2c052b4ec10710079fa7898a974e92e1: 2024-11-22T18:57:02,234 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-22T18:57:02,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:57:02,234 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 2c052b4ec10710079fa7898a974e92e1 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-22T18:57:02,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122809f9f9e14f2462fa3887e860a011946_2c052b4ec10710079fa7898a974e92e1 is 50, key is test_row_0/A:col10/1732301820905/Put/seqid=0 2024-11-22T18:57:02,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742148_1324 (size=12154) 2024-11-22T18:57:02,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:57:02,251 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122809f9f9e14f2462fa3887e860a011946_2c052b4ec10710079fa7898a974e92e1 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122809f9f9e14f2462fa3887e860a011946_2c052b4ec10710079fa7898a974e92e1 2024-11-22T18:57:02,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/A/51821a818f754e169e8067a12a0c7530, store: [table=TestAcidGuarantees family=A region=2c052b4ec10710079fa7898a974e92e1] 2024-11-22T18:57:02,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/A/51821a818f754e169e8067a12a0c7530 is 175, key is test_row_0/A:col10/1732301820905/Put/seqid=0 2024-11-22T18:57:02,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742149_1325 (size=30955) 2024-11-22T18:57:02,656 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/A/51821a818f754e169e8067a12a0c7530 2024-11-22T18:57:02,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/B/7901636f861044b4ad48563d8ef23557 is 50, key is test_row_0/B:col10/1732301820905/Put/seqid=0 2024-11-22T18:57:02,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742150_1326 (size=12001) 2024-11-22T18:57:02,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-22T18:57:03,069 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/B/7901636f861044b4ad48563d8ef23557 2024-11-22T18:57:03,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on 2c052b4ec10710079fa7898a974e92e1 2024-11-22T18:57:03,072 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. as already flushing 2024-11-22T18:57:03,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/C/c000d0ff843d4f31b91b7f0240989c45 is 50, key is test_row_0/C:col10/1732301820905/Put/seqid=0 2024-11-22T18:57:03,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742151_1327 (size=12001) 2024-11-22T18:57:03,084 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/C/c000d0ff843d4f31b91b7f0240989c45 2024-11-22T18:57:03,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/A/51821a818f754e169e8067a12a0c7530 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/51821a818f754e169e8067a12a0c7530 2024-11-22T18:57:03,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:03,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41730 deadline: 1732301883087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:03,094 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:03,094 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:03,094 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:03,095 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/51821a818f754e169e8067a12a0c7530, entries=150, sequenceid=41, filesize=30.2 K 2024-11-22T18:57:03,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:03,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41726 deadline: 1732301883092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:03,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/B/7901636f861044b4ad48563d8ef23557 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/7901636f861044b4ad48563d8ef23557 2024-11-22T18:57:03,097 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:03,097 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:03,097 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:03,099 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/7901636f861044b4ad48563d8ef23557, entries=150, sequenceid=41, filesize=11.7 K 2024-11-22T18:57:03,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:03,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732301883093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:03,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/C/c000d0ff843d4f31b91b7f0240989c45 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/c000d0ff843d4f31b91b7f0240989c45 2024-11-22T18:57:03,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:03,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:03,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:03,104 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/c000d0ff843d4f31b91b7f0240989c45, entries=150, sequenceid=41, filesize=11.7 K 2024-11-22T18:57:03,105 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 2c052b4ec10710079fa7898a974e92e1 in 871ms, sequenceid=41, compaction requested=false 2024-11-22T18:57:03,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 2c052b4ec10710079fa7898a974e92e1: 2024-11-22T18:57:03,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:57:03,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-22T18:57:03,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-22T18:57:03,108 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-22T18:57:03,108 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2540 sec 2024-11-22T18:57:03,110 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 2.2670 sec 2024-11-22T18:57:03,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on 2c052b4ec10710079fa7898a974e92e1 2024-11-22T18:57:03,130 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T18:57:03,130 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2c052b4ec10710079fa7898a974e92e1 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-22T18:57:03,140 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411223b3ffcac0c9340a4b482949c8f06fa43_2c052b4ec10710079fa7898a974e92e1 is 50, key is test_row_0/A:col10/1732301823105/Put/seqid=0 2024-11-22T18:57:03,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742152_1328 (size=17034) 2024-11-22T18:57:03,183 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:57:03,188 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411223b3ffcac0c9340a4b482949c8f06fa43_2c052b4ec10710079fa7898a974e92e1 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411223b3ffcac0c9340a4b482949c8f06fa43_2c052b4ec10710079fa7898a974e92e1 2024-11-22T18:57:03,190 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/A/2ec975cb39294402b430378b67f38ef6, store: [table=TestAcidGuarantees family=A region=2c052b4ec10710079fa7898a974e92e1] 2024-11-22T18:57:03,190 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/A/2ec975cb39294402b430378b67f38ef6 is 175, key is test_row_0/A:col10/1732301823105/Put/seqid=0 2024-11-22T18:57:03,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742153_1329 (size=48139) 2024-11-22T18:57:03,210 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=54, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/A/2ec975cb39294402b430378b67f38ef6 2024-11-22T18:57:03,222 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/B/2741c27e0aa041c3bc1ec62148a66e87 is 50, key is test_row_0/B:col10/1732301823105/Put/seqid=0 2024-11-22T18:57:03,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:03,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41730 deadline: 1732301883224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:03,237 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:03,237 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:03,237 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:03,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742154_1330 (size=12001) 2024-11-22T18:57:03,241 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/B/2741c27e0aa041c3bc1ec62148a66e87 2024-11-22T18:57:03,242 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:03,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732301883227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:03,242 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:03,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732301883230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:03,243 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:03,243 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:03,243 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:03,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:03,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41726 deadline: 1732301883232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:03,244 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:03,245 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:03,245 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:03,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:03,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732301883233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:03,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:03,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:03,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:03,252 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:03,252 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:03,252 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:03,254 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/C/ade2eda6d7c844589b9db95f2a6a77cf is 50, key is test_row_0/C:col10/1732301823105/Put/seqid=0 2024-11-22T18:57:03,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742155_1331 (size=12001) 2024-11-22T18:57:03,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:03,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41730 deadline: 1732301883346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:03,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:03,353 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:03,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732301883346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:03,353 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:03,353 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:03,353 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:03,353 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:03,353 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:03,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:03,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41726 deadline: 1732301883355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:03,366 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:03,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:03,366 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:03,366 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:03,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732301883355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:03,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:03,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:03,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:03,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:03,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732301883355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:03,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:03,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:03,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:03,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:03,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41730 deadline: 1732301883556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:03,564 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:03,564 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:03,564 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:03,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:03,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732301883565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:03,575 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:03,575 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:03,575 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:03,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:03,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41726 deadline: 1732301883575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:03,583 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:03,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:03,583 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:03,583 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:03,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732301883576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:03,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:03,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:03,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:03,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:03,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732301883576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:03,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:03,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:03,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:03,701 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/C/ade2eda6d7c844589b9db95f2a6a77cf 2024-11-22T18:57:03,706 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/A/2ec975cb39294402b430378b67f38ef6 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/2ec975cb39294402b430378b67f38ef6 2024-11-22T18:57:03,710 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/2ec975cb39294402b430378b67f38ef6, entries=250, sequenceid=54, filesize=47.0 K 2024-11-22T18:57:03,712 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/B/2741c27e0aa041c3bc1ec62148a66e87 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/2741c27e0aa041c3bc1ec62148a66e87 2024-11-22T18:57:03,716 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/2741c27e0aa041c3bc1ec62148a66e87, entries=150, sequenceid=54, filesize=11.7 K 2024-11-22T18:57:03,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/C/ade2eda6d7c844589b9db95f2a6a77cf as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/ade2eda6d7c844589b9db95f2a6a77cf 2024-11-22T18:57:03,720 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/ade2eda6d7c844589b9db95f2a6a77cf, entries=150, sequenceid=54, filesize=11.7 K 2024-11-22T18:57:03,721 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 2c052b4ec10710079fa7898a974e92e1 in 591ms, sequenceid=54, compaction requested=true 2024-11-22T18:57:03,721 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2c052b4ec10710079fa7898a974e92e1: 2024-11-22T18:57:03,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2c052b4ec10710079fa7898a974e92e1:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T18:57:03,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:57:03,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2c052b4ec10710079fa7898a974e92e1:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T18:57:03,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:57:03,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2c052b4ec10710079fa7898a974e92e1:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T18:57:03,722 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T18:57:03,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:57:03,722 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T18:57:03,723 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T18:57:03,723 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1541): 2c052b4ec10710079fa7898a974e92e1/A is initiating minor compaction (all files) 2024-11-22T18:57:03,723 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2c052b4ec10710079fa7898a974e92e1/A in TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:57:03,723 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T18:57:03,723 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/074178ce8a1b41f5abcc83b728726f37, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/51821a818f754e169e8067a12a0c7530, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/2ec975cb39294402b430378b67f38ef6] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp, totalSize=115.9 K 2024-11-22T18:57:03,723 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1541): 2c052b4ec10710079fa7898a974e92e1/B is initiating minor compaction (all files) 2024-11-22T18:57:03,723 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2c052b4ec10710079fa7898a974e92e1/B in TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:57:03,723 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:57:03,723 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. files: [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/074178ce8a1b41f5abcc83b728726f37, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/51821a818f754e169e8067a12a0c7530, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/2ec975cb39294402b430378b67f38ef6] 2024-11-22T18:57:03,724 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/5fbb26f6a2a8440789922d1f4fd3694a, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/7901636f861044b4ad48563d8ef23557, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/2741c27e0aa041c3bc1ec62148a66e87] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp, totalSize=35.2 K 2024-11-22T18:57:03,724 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 5fbb26f6a2a8440789922d1f4fd3694a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732301820853 2024-11-22T18:57:03,724 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 074178ce8a1b41f5abcc83b728726f37, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732301820853 2024-11-22T18:57:03,725 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 7901636f861044b4ad48563d8ef23557, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732301820891 2024-11-22T18:57:03,725 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51821a818f754e169e8067a12a0c7530, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732301820891 2024-11-22T18:57:03,725 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 2741c27e0aa041c3bc1ec62148a66e87, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732301823086 2024-11-22T18:57:03,725 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ec975cb39294402b430378b67f38ef6, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732301823086 2024-11-22T18:57:03,733 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2c052b4ec10710079fa7898a974e92e1] 2024-11-22T18:57:03,736 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2c052b4ec10710079fa7898a974e92e1#B#compaction#286 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:57:03,736 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/B/8a2dc37bdb2d41be8f55f6eed0dec53c is 50, key is test_row_0/B:col10/1732301823105/Put/seqid=0 2024-11-22T18:57:03,738 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241122e964fffd98df49feac3ede1bd626463a_2c052b4ec10710079fa7898a974e92e1 store=[table=TestAcidGuarantees family=A region=2c052b4ec10710079fa7898a974e92e1] 2024-11-22T18:57:03,740 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241122e964fffd98df49feac3ede1bd626463a_2c052b4ec10710079fa7898a974e92e1, store=[table=TestAcidGuarantees family=A region=2c052b4ec10710079fa7898a974e92e1] 2024-11-22T18:57:03,740 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122e964fffd98df49feac3ede1bd626463a_2c052b4ec10710079fa7898a974e92e1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2c052b4ec10710079fa7898a974e92e1] 2024-11-22T18:57:03,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742156_1332 (size=12104) 2024-11-22T18:57:03,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742157_1333 (size=4469) 2024-11-22T18:57:03,787 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2c052b4ec10710079fa7898a974e92e1#A#compaction#285 average throughput is 0.46 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:57:03,787 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/A/bcb26716e9794f35b7ae03c3beeb6659 is 175, key is test_row_0/A:col10/1732301823105/Put/seqid=0 2024-11-22T18:57:03,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742158_1334 (size=31058) 2024-11-22T18:57:03,807 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/A/bcb26716e9794f35b7ae03c3beeb6659 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/bcb26716e9794f35b7ae03c3beeb6659 2024-11-22T18:57:03,816 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2c052b4ec10710079fa7898a974e92e1/A of 2c052b4ec10710079fa7898a974e92e1 into bcb26716e9794f35b7ae03c3beeb6659(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:57:03,816 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2c052b4ec10710079fa7898a974e92e1: 2024-11-22T18:57:03,816 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., storeName=2c052b4ec10710079fa7898a974e92e1/A, priority=13, startTime=1732301823721; duration=0sec 2024-11-22T18:57:03,816 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:57:03,816 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2c052b4ec10710079fa7898a974e92e1:A 2024-11-22T18:57:03,816 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T18:57:03,818 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T18:57:03,818 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1541): 2c052b4ec10710079fa7898a974e92e1/C is initiating minor compaction (all files) 2024-11-22T18:57:03,819 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2c052b4ec10710079fa7898a974e92e1/C in TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:57:03,819 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/a85f58865dbb407bb7a45313653fd2d1, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/c000d0ff843d4f31b91b7f0240989c45, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/ade2eda6d7c844589b9db95f2a6a77cf] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp, totalSize=35.2 K 2024-11-22T18:57:03,819 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting a85f58865dbb407bb7a45313653fd2d1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732301820853 2024-11-22T18:57:03,820 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting c000d0ff843d4f31b91b7f0240989c45, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732301820891 2024-11-22T18:57:03,820 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting ade2eda6d7c844589b9db95f2a6a77cf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732301823086 2024-11-22T18:57:03,827 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2c052b4ec10710079fa7898a974e92e1#C#compaction#287 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:57:03,828 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/C/c78da3e492f64878b2195edac2649140 is 50, key is test_row_0/C:col10/1732301823105/Put/seqid=0 2024-11-22T18:57:03,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742159_1335 (size=12104) 2024-11-22T18:57:03,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on 2c052b4ec10710079fa7898a974e92e1 2024-11-22T18:57:03,883 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T18:57:03,883 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2c052b4ec10710079fa7898a974e92e1 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-22T18:57:03,897 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112294082aa1e81e43f6b384f225060e9023_2c052b4ec10710079fa7898a974e92e1 is 50, key is test_row_0/A:col10/1732301823881/Put/seqid=0 2024-11-22T18:57:03,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742160_1336 (size=14594) 2024-11-22T18:57:03,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:03,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41726 deadline: 1732301883905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:03,913 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:03,913 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:03,913 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:03,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:03,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732301883907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:03,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:03,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732301883908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:03,916 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:03,916 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:03,916 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:03,917 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:03,917 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:03,917 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:03,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:03,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732301883909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:03,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:03,922 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:03,922 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:03,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:03,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41730 deadline: 1732301883910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:03,922 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:03,922 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:03,922 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:04,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:04,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41726 deadline: 1732301884015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:04,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:04,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:04,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:04,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:04,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732301884026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:04,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:04,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732301884026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:04,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:04,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:04,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:04,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:04,034 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:04,034 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:04,034 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:04,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732301884026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:04,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:04,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:04,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:04,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:04,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41730 deadline: 1732301884026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:04,035 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:04,035 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:04,035 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:04,164 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/B/8a2dc37bdb2d41be8f55f6eed0dec53c as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/8a2dc37bdb2d41be8f55f6eed0dec53c 2024-11-22T18:57:04,168 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2c052b4ec10710079fa7898a974e92e1/B of 2c052b4ec10710079fa7898a974e92e1 into 8a2dc37bdb2d41be8f55f6eed0dec53c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:57:04,168 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2c052b4ec10710079fa7898a974e92e1: 2024-11-22T18:57:04,168 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., storeName=2c052b4ec10710079fa7898a974e92e1/B, priority=13, startTime=1732301823721; duration=0sec 2024-11-22T18:57:04,168 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:57:04,168 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2c052b4ec10710079fa7898a974e92e1:B 2024-11-22T18:57:04,239 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/C/c78da3e492f64878b2195edac2649140 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/c78da3e492f64878b2195edac2649140 2024-11-22T18:57:04,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:04,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41726 deadline: 1732301884236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:04,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:04,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:04,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:04,243 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2c052b4ec10710079fa7898a974e92e1/C of 2c052b4ec10710079fa7898a974e92e1 into c78da3e492f64878b2195edac2649140(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:57:04,243 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2c052b4ec10710079fa7898a974e92e1: 2024-11-22T18:57:04,243 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., storeName=2c052b4ec10710079fa7898a974e92e1/C, priority=13, startTime=1732301823722; duration=0sec 2024-11-22T18:57:04,243 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:57:04,243 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2c052b4ec10710079fa7898a974e92e1:C 2024-11-22T18:57:04,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:04,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732301884246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:04,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:04,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732301884246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:04,251 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:04,251 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:04,251 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:04,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:04,251 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:04,251 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:04,252 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:04,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732301884246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:04,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:04,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:04,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:04,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41730 deadline: 1732301884246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:04,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:04,252 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:04,252 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:04,252 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:04,304 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:57:04,310 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112294082aa1e81e43f6b384f225060e9023_2c052b4ec10710079fa7898a974e92e1 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112294082aa1e81e43f6b384f225060e9023_2c052b4ec10710079fa7898a974e92e1 2024-11-22T18:57:04,311 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/A/995c9296f3474981b0e84670de497e7e, store: [table=TestAcidGuarantees family=A region=2c052b4ec10710079fa7898a974e92e1] 2024-11-22T18:57:04,311 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/A/995c9296f3474981b0e84670de497e7e is 175, key is test_row_0/A:col10/1732301823881/Put/seqid=0 2024-11-22T18:57:04,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742161_1337 (size=39549) 2024-11-22T18:57:04,318 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=79, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/A/995c9296f3474981b0e84670de497e7e 2024-11-22T18:57:04,325 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/B/4fa0b94ef57e478b9b2e584235e93977 is 50, key is test_row_0/B:col10/1732301823881/Put/seqid=0 2024-11-22T18:57:04,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742162_1338 (size=12001) 2024-11-22T18:57:04,329 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/B/4fa0b94ef57e478b9b2e584235e93977 2024-11-22T18:57:04,336 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/C/2670acd0422a4c408c08d27f4d318701 is 50, key is test_row_0/C:col10/1732301823881/Put/seqid=0 2024-11-22T18:57:04,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742163_1339 (size=12001) 2024-11-22T18:57:04,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:04,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41726 deadline: 1732301884545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:04,552 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:04,552 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:04,552 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:04,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:04,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732301884556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:04,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:04,561 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:04,562 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:04,562 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:04,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732301884556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:04,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:04,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:04,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:04,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:04,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732301884556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:04,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:04,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:04,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:04,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:04,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41730 deadline: 1732301884565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:04,571 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:04,571 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:04,571 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:04,747 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/C/2670acd0422a4c408c08d27f4d318701 2024-11-22T18:57:04,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/A/995c9296f3474981b0e84670de497e7e as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/995c9296f3474981b0e84670de497e7e 2024-11-22T18:57:04,755 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/995c9296f3474981b0e84670de497e7e, entries=200, sequenceid=79, filesize=38.6 K 2024-11-22T18:57:04,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/B/4fa0b94ef57e478b9b2e584235e93977 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/4fa0b94ef57e478b9b2e584235e93977 2024-11-22T18:57:04,760 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/4fa0b94ef57e478b9b2e584235e93977, entries=150, sequenceid=79, filesize=11.7 K 2024-11-22T18:57:04,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/C/2670acd0422a4c408c08d27f4d318701 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/2670acd0422a4c408c08d27f4d318701 2024-11-22T18:57:04,765 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/2670acd0422a4c408c08d27f4d318701, entries=150, sequenceid=79, filesize=11.7 K 2024-11-22T18:57:04,766 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 2c052b4ec10710079fa7898a974e92e1 in 883ms, sequenceid=79, compaction requested=false 2024-11-22T18:57:04,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2c052b4ec10710079fa7898a974e92e1: 2024-11-22T18:57:05,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-22T18:57:05,006 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T18:57:05,006 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T18:57:05,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-22T18:57:05,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-22T18:57:05,008 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T18:57:05,009 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T18:57:05,009 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T18:57:05,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on 2c052b4ec10710079fa7898a974e92e1 2024-11-22T18:57:05,058 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T18:57:05,058 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2c052b4ec10710079fa7898a974e92e1 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-22T18:57:05,064 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112214e4d10b7574497599b3b43e19c424fe_2c052b4ec10710079fa7898a974e92e1 is 50, key is test_row_0/A:col10/1732301825056/Put/seqid=0 2024-11-22T18:57:05,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742164_1340 (size=14594) 2024-11-22T18:57:05,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:05,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732301885101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:05,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:05,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:05,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:05,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:05,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732301885103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:05,114 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:05,114 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:05,114 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:05,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-22T18:57:05,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:05,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732301885104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:05,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:05,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:05,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:05,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:05,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41730 deadline: 1732301885110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:05,120 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:05,120 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:05,120 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:05,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:05,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41726 deadline: 1732301885111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:05,124 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:05,124 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:05,124 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:05,161 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-22T18:57:05,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:57:05,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. as already flushing 2024-11-22T18:57:05,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:57:05,162 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 2c052b4ec10710079fa7898a974e92e1, NAME => 'TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:05,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 2c052b4ec10710079fa7898a974e92e1, NAME => 'TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:05,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2c052b4ec10710079fa7898a974e92e1, NAME => 'TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2c052b4ec10710079fa7898a974e92e1, NAME => 'TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:05,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:05,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732301885226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:05,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:05,234 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:05,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732301885226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:05,234 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:05,234 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:05,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:05,234 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:05,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732301885226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:05,235 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:05,235 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:05,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:05,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:05,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:05,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:05,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41730 deadline: 1732301885226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:05,235 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:05,235 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:05,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:05,235 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:05,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41726 deadline: 1732301885226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:05,235 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:05,235 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:05,235 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:05,314 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-22T18:57:05,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:57:05,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. as already flushing 2024-11-22T18:57:05,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:57:05,315 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 2c052b4ec10710079fa7898a974e92e1, NAME => 'TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:05,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 2c052b4ec10710079fa7898a974e92e1, NAME => 'TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:05,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2c052b4ec10710079fa7898a974e92e1, NAME => 'TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2c052b4ec10710079fa7898a974e92e1, NAME => 'TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:05,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-22T18:57:05,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:05,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732301885445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:05,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:05,457 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:05,457 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:05,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732301885445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:05,457 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:05,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:05,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:05,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:05,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:05,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732301885445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:05,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:05,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41730 deadline: 1732301885445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:05,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:05,458 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:05,458 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:05,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41726 deadline: 1732301885445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:05,458 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:05,458 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:05,458 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:05,458 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:05,458 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:05,458 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:05,458 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:05,467 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-22T18:57:05,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:57:05,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. as already flushing 2024-11-22T18:57:05,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:57:05,467 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 2c052b4ec10710079fa7898a974e92e1, NAME => 'TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:05,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 2c052b4ec10710079fa7898a974e92e1, NAME => 'TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:05,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2c052b4ec10710079fa7898a974e92e1, NAME => 'TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2c052b4ec10710079fa7898a974e92e1, NAME => 'TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:05,470 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:57:05,474 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112214e4d10b7574497599b3b43e19c424fe_2c052b4ec10710079fa7898a974e92e1 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112214e4d10b7574497599b3b43e19c424fe_2c052b4ec10710079fa7898a974e92e1 2024-11-22T18:57:05,474 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/A/67bb22be1ee8445bb25cb252c2a210f0, store: [table=TestAcidGuarantees family=A region=2c052b4ec10710079fa7898a974e92e1] 2024-11-22T18:57:05,475 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/A/67bb22be1ee8445bb25cb252c2a210f0 is 175, key is test_row_0/A:col10/1732301825056/Put/seqid=0 2024-11-22T18:57:05,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742165_1341 (size=39549) 2024-11-22T18:57:05,619 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-22T18:57:05,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:57:05,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. as already flushing 2024-11-22T18:57:05,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:57:05,620 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 2c052b4ec10710079fa7898a974e92e1, NAME => 'TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:05,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 2c052b4ec10710079fa7898a974e92e1, NAME => 'TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:05,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2c052b4ec10710079fa7898a974e92e1, NAME => 'TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2c052b4ec10710079fa7898a974e92e1, NAME => 'TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:05,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-22T18:57:05,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:05,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732301885766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:05,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:05,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:05,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:05,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732301885766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:05,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:05,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:05,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:05,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:05,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:05,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41730 deadline: 1732301885766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:05,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:05,771 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:05,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732301885766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:05,771 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:05,771 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:05,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:05,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:05,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41726 deadline: 1732301885766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:05,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:05,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:05,772 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:05,772 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:05,772 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:05,772 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-22T18:57:05,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:57:05,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. as already flushing 2024-11-22T18:57:05,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:57:05,773 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 2c052b4ec10710079fa7898a974e92e1, NAME => 'TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:05,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 2c052b4ec10710079fa7898a974e92e1, NAME => 'TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:05,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2c052b4ec10710079fa7898a974e92e1, NAME => 'TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2c052b4ec10710079fa7898a974e92e1, NAME => 'TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:05,879 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=94, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/A/67bb22be1ee8445bb25cb252c2a210f0 2024-11-22T18:57:05,886 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/B/39e585b55be7450bac954e7782b637dc is 50, key is test_row_0/B:col10/1732301825056/Put/seqid=0 2024-11-22T18:57:05,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742166_1342 (size=12001) 2024-11-22T18:57:05,925 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-22T18:57:05,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:57:05,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. as already flushing 2024-11-22T18:57:05,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:57:05,926 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 2c052b4ec10710079fa7898a974e92e1, NAME => 'TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:05,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 2c052b4ec10710079fa7898a974e92e1, NAME => 'TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:05,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2c052b4ec10710079fa7898a974e92e1, NAME => 'TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2c052b4ec10710079fa7898a974e92e1, NAME => 'TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:06,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-22T18:57:06,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:57:06,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. as already flushing 2024-11-22T18:57:06,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:57:06,079 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 2c052b4ec10710079fa7898a974e92e1, NAME => 'TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:06,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 2c052b4ec10710079fa7898a974e92e1, NAME => 'TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:06,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2c052b4ec10710079fa7898a974e92e1, NAME => 'TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2c052b4ec10710079fa7898a974e92e1, NAME => 'TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:06,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-22T18:57:06,232 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-22T18:57:06,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:57:06,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. as already flushing 2024-11-22T18:57:06,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:57:06,232 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 2c052b4ec10710079fa7898a974e92e1, NAME => 'TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:06,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 2c052b4ec10710079fa7898a974e92e1, NAME => 'TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:06,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2c052b4ec10710079fa7898a974e92e1, NAME => 'TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2c052b4ec10710079fa7898a974e92e1, NAME => 'TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:06,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:06,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732301886275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:06,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:06,283 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:06,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41730 deadline: 1732301886275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:06,283 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:06,283 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:06,283 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:06,283 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:06,283 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:06,292 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/B/39e585b55be7450bac954e7782b637dc 2024-11-22T18:57:06,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:06,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732301886286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:06,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:06,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:06,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:06,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:06,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732301886286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:06,296 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:06,296 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:06,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:06,296 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:06,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41726 deadline: 1732301886286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:06,297 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:06,297 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:06,297 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:06,299 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/C/301a8d024ce549c2b8f2c9136b43eff9 is 50, key is test_row_0/C:col10/1732301825056/Put/seqid=0 2024-11-22T18:57:06,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742167_1343 (size=12001) 2024-11-22T18:57:06,385 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-22T18:57:06,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:57:06,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. as already flushing 2024-11-22T18:57:06,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:57:06,385 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 2c052b4ec10710079fa7898a974e92e1, NAME => 'TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:06,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 2c052b4ec10710079fa7898a974e92e1, NAME => 'TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:06,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2c052b4ec10710079fa7898a974e92e1, NAME => 'TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2c052b4ec10710079fa7898a974e92e1, NAME => 'TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:06,538 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-22T18:57:06,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:57:06,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. as already flushing 2024-11-22T18:57:06,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:57:06,539 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 2c052b4ec10710079fa7898a974e92e1, NAME => 'TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:06,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 2c052b4ec10710079fa7898a974e92e1, NAME => 'TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:06,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2c052b4ec10710079fa7898a974e92e1, NAME => 'TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2c052b4ec10710079fa7898a974e92e1, NAME => 'TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:06,690 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-22T18:57:06,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:57:06,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. as already flushing 2024-11-22T18:57:06,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:57:06,691 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 2c052b4ec10710079fa7898a974e92e1, NAME => 'TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:06,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 2c052b4ec10710079fa7898a974e92e1, NAME => 'TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:06,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2c052b4ec10710079fa7898a974e92e1, NAME => 'TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2c052b4ec10710079fa7898a974e92e1, NAME => 'TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:06,706 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/C/301a8d024ce549c2b8f2c9136b43eff9 2024-11-22T18:57:06,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/A/67bb22be1ee8445bb25cb252c2a210f0 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/67bb22be1ee8445bb25cb252c2a210f0 2024-11-22T18:57:06,719 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/67bb22be1ee8445bb25cb252c2a210f0, entries=200, sequenceid=94, filesize=38.6 K 2024-11-22T18:57:06,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/B/39e585b55be7450bac954e7782b637dc as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/39e585b55be7450bac954e7782b637dc 2024-11-22T18:57:06,729 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/39e585b55be7450bac954e7782b637dc, entries=150, sequenceid=94, filesize=11.7 K 2024-11-22T18:57:06,730 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/C/301a8d024ce549c2b8f2c9136b43eff9 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/301a8d024ce549c2b8f2c9136b43eff9 2024-11-22T18:57:06,735 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/301a8d024ce549c2b8f2c9136b43eff9, entries=150, sequenceid=94, filesize=11.7 K 2024-11-22T18:57:06,735 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 2c052b4ec10710079fa7898a974e92e1 in 1677ms, sequenceid=94, compaction requested=true 2024-11-22T18:57:06,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2c052b4ec10710079fa7898a974e92e1: 2024-11-22T18:57:06,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2c052b4ec10710079fa7898a974e92e1:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T18:57:06,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:57:06,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2c052b4ec10710079fa7898a974e92e1:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T18:57:06,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:57:06,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2c052b4ec10710079fa7898a974e92e1:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T18:57:06,736 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T18:57:06,736 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T18:57:06,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:57:06,737 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T18:57:06,737 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110156 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T18:57:06,737 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1541): 2c052b4ec10710079fa7898a974e92e1/B is initiating minor compaction (all files) 2024-11-22T18:57:06,737 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1541): 2c052b4ec10710079fa7898a974e92e1/A is initiating minor compaction (all files) 2024-11-22T18:57:06,737 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2c052b4ec10710079fa7898a974e92e1/B in TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:57:06,737 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2c052b4ec10710079fa7898a974e92e1/A in TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:57:06,737 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/8a2dc37bdb2d41be8f55f6eed0dec53c, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/4fa0b94ef57e478b9b2e584235e93977, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/39e585b55be7450bac954e7782b637dc] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp, totalSize=35.3 K 2024-11-22T18:57:06,737 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/bcb26716e9794f35b7ae03c3beeb6659, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/995c9296f3474981b0e84670de497e7e, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/67bb22be1ee8445bb25cb252c2a210f0] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp, totalSize=107.6 K 2024-11-22T18:57:06,737 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:57:06,737 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. files: [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/bcb26716e9794f35b7ae03c3beeb6659, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/995c9296f3474981b0e84670de497e7e, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/67bb22be1ee8445bb25cb252c2a210f0] 2024-11-22T18:57:06,738 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting bcb26716e9794f35b7ae03c3beeb6659, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732301823086 2024-11-22T18:57:06,738 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a2dc37bdb2d41be8f55f6eed0dec53c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732301823086 2024-11-22T18:57:06,738 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 4fa0b94ef57e478b9b2e584235e93977, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732301823220 2024-11-22T18:57:06,738 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 39e585b55be7450bac954e7782b637dc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732301823907 2024-11-22T18:57:06,739 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 995c9296f3474981b0e84670de497e7e, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732301823220 2024-11-22T18:57:06,739 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67bb22be1ee8445bb25cb252c2a210f0, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732301823894 2024-11-22T18:57:06,747 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2c052b4ec10710079fa7898a974e92e1] 2024-11-22T18:57:06,758 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2c052b4ec10710079fa7898a974e92e1#B#compaction#295 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:57:06,759 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/B/08bd76133ee145da8aa88a0d179038f6 is 50, key is test_row_0/B:col10/1732301825056/Put/seqid=0 2024-11-22T18:57:06,759 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241122d4a5f16c558f4aa1b6a24613bbd34d6e_2c052b4ec10710079fa7898a974e92e1 store=[table=TestAcidGuarantees family=A region=2c052b4ec10710079fa7898a974e92e1] 2024-11-22T18:57:06,761 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241122d4a5f16c558f4aa1b6a24613bbd34d6e_2c052b4ec10710079fa7898a974e92e1, store=[table=TestAcidGuarantees family=A region=2c052b4ec10710079fa7898a974e92e1] 2024-11-22T18:57:06,761 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122d4a5f16c558f4aa1b6a24613bbd34d6e_2c052b4ec10710079fa7898a974e92e1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2c052b4ec10710079fa7898a974e92e1] 2024-11-22T18:57:06,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742168_1344 (size=12207) 2024-11-22T18:57:06,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742169_1345 (size=4469) 2024-11-22T18:57:06,845 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-22T18:57:06,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:57:06,845 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 2c052b4ec10710079fa7898a974e92e1 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-22T18:57:06,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112264ac991c44904684bea720d28f980841_2c052b4ec10710079fa7898a974e92e1 is 50, key is test_row_0/A:col10/1732301825110/Put/seqid=0 2024-11-22T18:57:06,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742170_1346 (size=12154) 2024-11-22T18:57:06,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:57:06,860 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112264ac991c44904684bea720d28f980841_2c052b4ec10710079fa7898a974e92e1 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112264ac991c44904684bea720d28f980841_2c052b4ec10710079fa7898a974e92e1 2024-11-22T18:57:06,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/A/a1e2f7dfc56d4ed2b3156627df25e7fa, store: [table=TestAcidGuarantees family=A region=2c052b4ec10710079fa7898a974e92e1] 2024-11-22T18:57:06,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/A/a1e2f7dfc56d4ed2b3156627df25e7fa is 175, key is test_row_0/A:col10/1732301825110/Put/seqid=0 2024-11-22T18:57:06,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742171_1347 (size=30955) 2024-11-22T18:57:06,868 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=117, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/A/a1e2f7dfc56d4ed2b3156627df25e7fa 2024-11-22T18:57:06,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/B/2f0997c1d44f4cf58a129f56a5719454 is 50, key is test_row_0/B:col10/1732301825110/Put/seqid=0 2024-11-22T18:57:06,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742172_1348 (size=12001) 2024-11-22T18:57:07,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-22T18:57:07,201 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/B/08bd76133ee145da8aa88a0d179038f6 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/08bd76133ee145da8aa88a0d179038f6 2024-11-22T18:57:07,206 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2c052b4ec10710079fa7898a974e92e1/B of 2c052b4ec10710079fa7898a974e92e1 into 08bd76133ee145da8aa88a0d179038f6(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:57:07,206 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2c052b4ec10710079fa7898a974e92e1: 2024-11-22T18:57:07,206 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., storeName=2c052b4ec10710079fa7898a974e92e1/B, priority=13, startTime=1732301826736; duration=0sec 2024-11-22T18:57:07,206 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:57:07,206 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2c052b4ec10710079fa7898a974e92e1:B 2024-11-22T18:57:07,206 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T18:57:07,207 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T18:57:07,207 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1541): 2c052b4ec10710079fa7898a974e92e1/C is initiating minor compaction (all files) 2024-11-22T18:57:07,207 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2c052b4ec10710079fa7898a974e92e1/C in TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:57:07,207 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/c78da3e492f64878b2195edac2649140, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/2670acd0422a4c408c08d27f4d318701, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/301a8d024ce549c2b8f2c9136b43eff9] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp, totalSize=35.3 K 2024-11-22T18:57:07,207 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2c052b4ec10710079fa7898a974e92e1#A#compaction#294 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:57:07,208 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting c78da3e492f64878b2195edac2649140, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732301823086 2024-11-22T18:57:07,208 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 2670acd0422a4c408c08d27f4d318701, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732301823220 2024-11-22T18:57:07,208 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/A/6eac306947ac4cc49139bc41e5b55c17 is 175, key is test_row_0/A:col10/1732301825056/Put/seqid=0 2024-11-22T18:57:07,208 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 301a8d024ce549c2b8f2c9136b43eff9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732301823907 2024-11-22T18:57:07,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742173_1349 (size=31161) 2024-11-22T18:57:07,218 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/A/6eac306947ac4cc49139bc41e5b55c17 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/6eac306947ac4cc49139bc41e5b55c17 2024-11-22T18:57:07,222 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2c052b4ec10710079fa7898a974e92e1/A of 2c052b4ec10710079fa7898a974e92e1 into 6eac306947ac4cc49139bc41e5b55c17(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:57:07,222 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2c052b4ec10710079fa7898a974e92e1: 2024-11-22T18:57:07,222 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., storeName=2c052b4ec10710079fa7898a974e92e1/A, priority=13, startTime=1732301826736; duration=0sec 2024-11-22T18:57:07,222 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:57:07,222 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2c052b4ec10710079fa7898a974e92e1#C#compaction#298 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:57:07,222 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2c052b4ec10710079fa7898a974e92e1:A 2024-11-22T18:57:07,223 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/C/a0d68050bed74aa9a2e2634105a3b22d is 50, key is test_row_0/C:col10/1732301825056/Put/seqid=0 2024-11-22T18:57:07,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742174_1350 (size=12207) 2024-11-22T18:57:07,230 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/C/a0d68050bed74aa9a2e2634105a3b22d as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/a0d68050bed74aa9a2e2634105a3b22d 2024-11-22T18:57:07,234 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2c052b4ec10710079fa7898a974e92e1/C of 2c052b4ec10710079fa7898a974e92e1 into a0d68050bed74aa9a2e2634105a3b22d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:57:07,234 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2c052b4ec10710079fa7898a974e92e1: 2024-11-22T18:57:07,234 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., storeName=2c052b4ec10710079fa7898a974e92e1/C, priority=13, startTime=1732301826736; duration=0sec 2024-11-22T18:57:07,234 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:57:07,234 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2c052b4ec10710079fa7898a974e92e1:C 2024-11-22T18:57:07,283 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/B/2f0997c1d44f4cf58a129f56a5719454 2024-11-22T18:57:07,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/C/3dbc0d22f2a546c2b3fd162491ce08b0 is 50, key is test_row_0/C:col10/1732301825110/Put/seqid=0 2024-11-22T18:57:07,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on 2c052b4ec10710079fa7898a974e92e1 2024-11-22T18:57:07,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. as already flushing 2024-11-22T18:57:07,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742175_1351 (size=12001) 2024-11-22T18:57:07,311 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/C/3dbc0d22f2a546c2b3fd162491ce08b0 2024-11-22T18:57:07,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/A/a1e2f7dfc56d4ed2b3156627df25e7fa as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/a1e2f7dfc56d4ed2b3156627df25e7fa 2024-11-22T18:57:07,319 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/a1e2f7dfc56d4ed2b3156627df25e7fa, entries=150, sequenceid=117, filesize=30.2 K 2024-11-22T18:57:07,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/B/2f0997c1d44f4cf58a129f56a5719454 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/2f0997c1d44f4cf58a129f56a5719454 2024-11-22T18:57:07,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:07,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41726 deadline: 1732301887314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:07,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:07,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:07,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:07,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732301887319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:07,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:07,330 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:07,330 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:07,330 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:07,331 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/2f0997c1d44f4cf58a129f56a5719454, entries=150, sequenceid=117, filesize=11.7 K 2024-11-22T18:57:07,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/C/3dbc0d22f2a546c2b3fd162491ce08b0 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/3dbc0d22f2a546c2b3fd162491ce08b0 2024-11-22T18:57:07,338 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/3dbc0d22f2a546c2b3fd162491ce08b0, entries=150, sequenceid=117, filesize=11.7 K 2024-11-22T18:57:07,339 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for 2c052b4ec10710079fa7898a974e92e1 in 494ms, sequenceid=117, compaction requested=false 2024-11-22T18:57:07,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 2c052b4ec10710079fa7898a974e92e1: 2024-11-22T18:57:07,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:57:07,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-22T18:57:07,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-22T18:57:07,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on 2c052b4ec10710079fa7898a974e92e1 2024-11-22T18:57:07,342 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T18:57:07,342 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2c052b4ec10710079fa7898a974e92e1 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-22T18:57:07,346 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-22T18:57:07,346 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3350 sec 2024-11-22T18:57:07,348 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 2.3410 sec 2024-11-22T18:57:07,355 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411225d3c19caa1a3435eaefc2880cfb5d5e1_2c052b4ec10710079fa7898a974e92e1 is 50, key is test_row_0/A:col10/1732301827340/Put/seqid=0 2024-11-22T18:57:07,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742176_1352 (size=17284) 2024-11-22T18:57:07,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:07,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41730 deadline: 1732301887398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:07,403 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:07,403 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:07,403 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:07,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:07,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732301887399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:07,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:07,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:07,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:07,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:07,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732301887400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:07,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:07,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:07,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:07,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:07,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41726 deadline: 1732301887435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:07,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:07,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732301887435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:07,441 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:07,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:07,441 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:07,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:07,441 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:07,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:07,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:07,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41730 deadline: 1732301887505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:07,514 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:07,514 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:07,514 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:07,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:07,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732301887515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:07,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:07,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:07,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732301887515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:07,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:07,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:07,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:07,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:07,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:07,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:07,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732301887645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:07,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:07,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:07,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:07,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:07,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41726 deadline: 1732301887646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:07,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:07,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:07,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:07,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:07,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41730 deadline: 1732301887725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:07,735 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:07,736 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:07,736 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:07,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:07,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732301887736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:07,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:07,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:07,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:07,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:07,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732301887736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:07,740 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:07,740 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:07,740 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:07,764 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:57:07,770 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411225d3c19caa1a3435eaefc2880cfb5d5e1_2c052b4ec10710079fa7898a974e92e1 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411225d3c19caa1a3435eaefc2880cfb5d5e1_2c052b4ec10710079fa7898a974e92e1 2024-11-22T18:57:07,771 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/A/82c66899e3ab42caa9e66df18ba7db40, store: [table=TestAcidGuarantees family=A region=2c052b4ec10710079fa7898a974e92e1] 2024-11-22T18:57:07,772 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/A/82c66899e3ab42caa9e66df18ba7db40 is 175, key is test_row_0/A:col10/1732301827340/Put/seqid=0 2024-11-22T18:57:07,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742177_1353 (size=48389) 2024-11-22T18:57:07,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:07,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732301887966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:07,975 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:07,975 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:07,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:07,975 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:07,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41726 deadline: 1732301887966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:07,975 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:07,975 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:07,975 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:08,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:08,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732301888046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:08,049 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:08,049 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:08,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:08,049 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:08,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41730 deadline: 1732301888046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:08,050 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:08,050 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:08,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:08,050 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:08,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732301888046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:08,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:08,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:08,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:08,185 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=136, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/A/82c66899e3ab42caa9e66df18ba7db40 2024-11-22T18:57:08,192 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/B/efb65377ffd54875b3fc2a0f942ff147 is 50, key is test_row_0/B:col10/1732301827340/Put/seqid=0 2024-11-22T18:57:08,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742178_1354 (size=12151) 2024-11-22T18:57:08,202 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/B/efb65377ffd54875b3fc2a0f942ff147 2024-11-22T18:57:08,210 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/C/9611cffd6f444e9198538a9aeb08285a is 50, key is test_row_0/C:col10/1732301827340/Put/seqid=0 2024-11-22T18:57:08,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742179_1355 (size=12151) 2024-11-22T18:57:08,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:08,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732301888486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:08,492 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:08,492 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:08,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:08,492 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:08,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41726 deadline: 1732301888486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:08,493 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:08,493 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:08,493 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:08,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:08,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732301888556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:08,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:08,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:08,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:08,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41730 deadline: 1732301888556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:08,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:08,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:08,562 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:08,562 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:08,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732301888556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:08,562 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:08,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:08,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:08,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:08,618 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/C/9611cffd6f444e9198538a9aeb08285a 2024-11-22T18:57:08,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/A/82c66899e3ab42caa9e66df18ba7db40 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/82c66899e3ab42caa9e66df18ba7db40 2024-11-22T18:57:08,626 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/82c66899e3ab42caa9e66df18ba7db40, entries=250, sequenceid=136, filesize=47.3 K 2024-11-22T18:57:08,627 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/B/efb65377ffd54875b3fc2a0f942ff147 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/efb65377ffd54875b3fc2a0f942ff147 2024-11-22T18:57:08,630 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/efb65377ffd54875b3fc2a0f942ff147, entries=150, sequenceid=136, filesize=11.9 K 2024-11-22T18:57:08,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/C/9611cffd6f444e9198538a9aeb08285a as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/9611cffd6f444e9198538a9aeb08285a 2024-11-22T18:57:08,634 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/9611cffd6f444e9198538a9aeb08285a, entries=150, sequenceid=136, filesize=11.9 K 2024-11-22T18:57:08,635 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 2c052b4ec10710079fa7898a974e92e1 in 1292ms, sequenceid=136, compaction requested=true 2024-11-22T18:57:08,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2c052b4ec10710079fa7898a974e92e1: 2024-11-22T18:57:08,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2c052b4ec10710079fa7898a974e92e1:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T18:57:08,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:57:08,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2c052b4ec10710079fa7898a974e92e1:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T18:57:08,635 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T18:57:08,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:57:08,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2c052b4ec10710079fa7898a974e92e1:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T18:57:08,635 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T18:57:08,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:57:08,636 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110505 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T18:57:08,636 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T18:57:08,636 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1541): 2c052b4ec10710079fa7898a974e92e1/A is initiating minor compaction (all files) 2024-11-22T18:57:08,636 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1541): 2c052b4ec10710079fa7898a974e92e1/B is initiating minor compaction (all files) 2024-11-22T18:57:08,636 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2c052b4ec10710079fa7898a974e92e1/A in TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:57:08,636 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2c052b4ec10710079fa7898a974e92e1/B in TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:57:08,636 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/6eac306947ac4cc49139bc41e5b55c17, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/a1e2f7dfc56d4ed2b3156627df25e7fa, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/82c66899e3ab42caa9e66df18ba7db40] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp, totalSize=107.9 K 2024-11-22T18:57:08,636 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/08bd76133ee145da8aa88a0d179038f6, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/2f0997c1d44f4cf58a129f56a5719454, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/efb65377ffd54875b3fc2a0f942ff147] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp, totalSize=35.5 K 2024-11-22T18:57:08,636 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:57:08,636 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. files: [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/6eac306947ac4cc49139bc41e5b55c17, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/a1e2f7dfc56d4ed2b3156627df25e7fa, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/82c66899e3ab42caa9e66df18ba7db40] 2024-11-22T18:57:08,636 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 08bd76133ee145da8aa88a0d179038f6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732301823907 2024-11-22T18:57:08,636 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6eac306947ac4cc49139bc41e5b55c17, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732301823907 2024-11-22T18:57:08,636 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f0997c1d44f4cf58a129f56a5719454, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732301825101 2024-11-22T18:57:08,636 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1e2f7dfc56d4ed2b3156627df25e7fa, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732301825101 2024-11-22T18:57:08,637 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting efb65377ffd54875b3fc2a0f942ff147, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732301827327 2024-11-22T18:57:08,637 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82c66899e3ab42caa9e66df18ba7db40, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732301827317 2024-11-22T18:57:08,643 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2c052b4ec10710079fa7898a974e92e1] 2024-11-22T18:57:08,644 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2c052b4ec10710079fa7898a974e92e1#B#compaction#304 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:57:08,644 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/B/204abe18575d437387cd66015e375ee0 is 50, key is test_row_0/B:col10/1732301827340/Put/seqid=0 2024-11-22T18:57:08,645 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112242266337a3f14484b4b12ac1400fc29e_2c052b4ec10710079fa7898a974e92e1 store=[table=TestAcidGuarantees family=A region=2c052b4ec10710079fa7898a974e92e1] 2024-11-22T18:57:08,647 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112242266337a3f14484b4b12ac1400fc29e_2c052b4ec10710079fa7898a974e92e1, store=[table=TestAcidGuarantees family=A region=2c052b4ec10710079fa7898a974e92e1] 2024-11-22T18:57:08,647 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112242266337a3f14484b4b12ac1400fc29e_2c052b4ec10710079fa7898a974e92e1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2c052b4ec10710079fa7898a974e92e1] 2024-11-22T18:57:08,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742180_1356 (size=12459) 2024-11-22T18:57:08,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742181_1357 (size=4469) 2024-11-22T18:57:08,656 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2c052b4ec10710079fa7898a974e92e1#A#compaction#303 average throughput is 1.88 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:57:08,656 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/B/204abe18575d437387cd66015e375ee0 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/204abe18575d437387cd66015e375ee0 2024-11-22T18:57:08,656 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/A/425593a2767d4f50ad8d502ff2438f07 is 175, key is test_row_0/A:col10/1732301827340/Put/seqid=0 2024-11-22T18:57:08,661 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2c052b4ec10710079fa7898a974e92e1/B of 2c052b4ec10710079fa7898a974e92e1 into 204abe18575d437387cd66015e375ee0(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:57:08,661 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2c052b4ec10710079fa7898a974e92e1: 2024-11-22T18:57:08,661 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., storeName=2c052b4ec10710079fa7898a974e92e1/B, priority=13, startTime=1732301828635; duration=0sec 2024-11-22T18:57:08,661 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:57:08,661 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2c052b4ec10710079fa7898a974e92e1:B 2024-11-22T18:57:08,661 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T18:57:08,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742182_1358 (size=31413) 2024-11-22T18:57:08,662 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T18:57:08,662 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1541): 2c052b4ec10710079fa7898a974e92e1/C is initiating minor compaction (all files) 2024-11-22T18:57:08,662 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2c052b4ec10710079fa7898a974e92e1/C in TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:57:08,662 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/a0d68050bed74aa9a2e2634105a3b22d, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/3dbc0d22f2a546c2b3fd162491ce08b0, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/9611cffd6f444e9198538a9aeb08285a] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp, totalSize=35.5 K 2024-11-22T18:57:08,663 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting a0d68050bed74aa9a2e2634105a3b22d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732301823907 2024-11-22T18:57:08,663 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 3dbc0d22f2a546c2b3fd162491ce08b0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732301825101 2024-11-22T18:57:08,663 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 9611cffd6f444e9198538a9aeb08285a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732301827327 2024-11-22T18:57:08,666 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/A/425593a2767d4f50ad8d502ff2438f07 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/425593a2767d4f50ad8d502ff2438f07 2024-11-22T18:57:08,670 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2c052b4ec10710079fa7898a974e92e1#C#compaction#305 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:57:08,670 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/C/baec2ea185ae4f93b4d92b96fc1e4bad is 50, key is test_row_0/C:col10/1732301827340/Put/seqid=0 2024-11-22T18:57:08,673 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2c052b4ec10710079fa7898a974e92e1/A of 2c052b4ec10710079fa7898a974e92e1 into 425593a2767d4f50ad8d502ff2438f07(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:57:08,673 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2c052b4ec10710079fa7898a974e92e1: 2024-11-22T18:57:08,673 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., storeName=2c052b4ec10710079fa7898a974e92e1/A, priority=13, startTime=1732301828635; duration=0sec 2024-11-22T18:57:08,673 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:57:08,673 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2c052b4ec10710079fa7898a974e92e1:A 2024-11-22T18:57:08,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742183_1359 (size=12459) 2024-11-22T18:57:09,079 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/C/baec2ea185ae4f93b4d92b96fc1e4bad as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/baec2ea185ae4f93b4d92b96fc1e4bad 2024-11-22T18:57:09,085 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2c052b4ec10710079fa7898a974e92e1/C of 2c052b4ec10710079fa7898a974e92e1 into baec2ea185ae4f93b4d92b96fc1e4bad(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:57:09,085 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2c052b4ec10710079fa7898a974e92e1: 2024-11-22T18:57:09,085 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., storeName=2c052b4ec10710079fa7898a974e92e1/C, priority=13, startTime=1732301828635; duration=0sec 2024-11-22T18:57:09,085 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:57:09,085 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2c052b4ec10710079fa7898a974e92e1:C 2024-11-22T18:57:09,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-22T18:57:09,176 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T18:57:09,176 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T18:57:09,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-22T18:57:09,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-22T18:57:09,178 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T18:57:09,178 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T18:57:09,179 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T18:57:09,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-22T18:57:09,330 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-22T18:57:09,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:57:09,331 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing 2c052b4ec10710079fa7898a974e92e1 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-22T18:57:09,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112210270a9fe7644aa0910be31f0c87808e_2c052b4ec10710079fa7898a974e92e1 is 50, key is test_row_0/A:col10/1732301827397/Put/seqid=0 2024-11-22T18:57:09,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742184_1360 (size=12304) 2024-11-22T18:57:09,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-22T18:57:09,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on 2c052b4ec10710079fa7898a974e92e1 2024-11-22T18:57:09,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. as already flushing 2024-11-22T18:57:09,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:09,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732301889576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:09,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:09,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41730 deadline: 1732301889576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:09,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:09,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732301889576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:09,584 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to d79ba0c344fb,43593,1732301724173 for 'test_row_1' in 2c052b4ec10710079fa7898a974e92e1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] ... 32 more 2024-11-22T18:57:09,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:09,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:09,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:09,585 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to d79ba0c344fb,43593,1732301724173 for 'test_row_0' in 2c052b4ec10710079fa7898a974e92e1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] ... 32 more 2024-11-22T18:57:09,585 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:09,585 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:09,585 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:09,585 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to d79ba0c344fb,43593,1732301724173 for 'test_row_2' in 2c052b4ec10710079fa7898a974e92e1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] ... 32 more 2024-11-22T18:57:09,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:09,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:09,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:09,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:09,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41726 deadline: 1732301889581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:09,588 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:09,588 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:09,588 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:09,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:09,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732301889584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:09,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:09,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:09,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:09,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:09,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41726 deadline: 1732301889697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:09,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:09,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:09,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:09,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732301889697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:09,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:09,702 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:09,702 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:09,702 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:09,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:57:09,745 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112210270a9fe7644aa0910be31f0c87808e_2c052b4ec10710079fa7898a974e92e1 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112210270a9fe7644aa0910be31f0c87808e_2c052b4ec10710079fa7898a974e92e1 2024-11-22T18:57:09,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/A/c435fab400aa4a79bc7b0aa0a6cb0f2a, store: [table=TestAcidGuarantees family=A region=2c052b4ec10710079fa7898a974e92e1] 2024-11-22T18:57:09,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/A/c435fab400aa4a79bc7b0aa0a6cb0f2a is 175, key is test_row_0/A:col10/1732301827397/Put/seqid=0 2024-11-22T18:57:09,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742185_1361 (size=31105) 2024-11-22T18:57:09,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-22T18:57:09,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:09,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41726 deadline: 1732301889905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:09,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:09,913 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:09,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732301889905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:09,913 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:09,913 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:09,914 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:09,914 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:09,914 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:10,150 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=159, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/A/c435fab400aa4a79bc7b0aa0a6cb0f2a 2024-11-22T18:57:10,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/B/7814d8b792274856bc929285dc1d8a3b is 50, key is test_row_0/B:col10/1732301827397/Put/seqid=0 2024-11-22T18:57:10,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742186_1362 (size=12151) 2024-11-22T18:57:10,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:10,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732301890216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:10,223 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:10,223 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:10,223 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:10,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:10,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41726 deadline: 1732301890226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:10,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:10,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:10,234 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:10,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-22T18:57:10,562 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/B/7814d8b792274856bc929285dc1d8a3b 2024-11-22T18:57:10,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/C/80341b9241a244418395e20b5f1e9ffc is 50, key is test_row_0/C:col10/1732301827397/Put/seqid=0 2024-11-22T18:57:10,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742187_1363 (size=12151) 2024-11-22T18:57:10,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:10,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732301890725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:10,729 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:10,729 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:10,729 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:10,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:10,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41726 deadline: 1732301890746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:10,749 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:10,749 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:10,749 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:10,974 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/C/80341b9241a244418395e20b5f1e9ffc 2024-11-22T18:57:10,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/A/c435fab400aa4a79bc7b0aa0a6cb0f2a as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/c435fab400aa4a79bc7b0aa0a6cb0f2a 2024-11-22T18:57:10,982 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/c435fab400aa4a79bc7b0aa0a6cb0f2a, entries=150, sequenceid=159, filesize=30.4 K 2024-11-22T18:57:10,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/B/7814d8b792274856bc929285dc1d8a3b as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/7814d8b792274856bc929285dc1d8a3b 2024-11-22T18:57:10,989 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/7814d8b792274856bc929285dc1d8a3b, entries=150, sequenceid=159, filesize=11.9 K 2024-11-22T18:57:10,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/C/80341b9241a244418395e20b5f1e9ffc as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/80341b9241a244418395e20b5f1e9ffc 2024-11-22T18:57:10,994 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/80341b9241a244418395e20b5f1e9ffc, entries=150, sequenceid=159, filesize=11.9 K 2024-11-22T18:57:10,995 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 2c052b4ec10710079fa7898a974e92e1 in 1664ms, sequenceid=159, compaction requested=false 2024-11-22T18:57:10,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for 2c052b4ec10710079fa7898a974e92e1: 2024-11-22T18:57:10,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:57:10,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-22T18:57:10,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-22T18:57:10,998 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-22T18:57:10,998 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8180 sec 2024-11-22T18:57:11,000 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 1.8230 sec 2024-11-22T18:57:11,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-22T18:57:11,326 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T18:57:11,326 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T18:57:11,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-22T18:57:11,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-22T18:57:11,328 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T18:57:11,329 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T18:57:11,329 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T18:57:11,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-22T18:57:11,481 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-22T18:57:11,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:57:11,482 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing 2c052b4ec10710079fa7898a974e92e1 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-22T18:57:11,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411226055c7d005fd45a4a692601c03f3a48e_2c052b4ec10710079fa7898a974e92e1 is 50, key is test_row_0/A:col10/1732301829583/Put/seqid=0 2024-11-22T18:57:11,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742188_1364 (size=12304) 2024-11-22T18:57:11,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on 2c052b4ec10710079fa7898a974e92e1 2024-11-22T18:57:11,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. as already flushing 2024-11-22T18:57:11,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-22T18:57:11,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:11,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732301891650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:11,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:11,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:11,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:11,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:11,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732301891651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:11,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:11,662 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:11,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41730 deadline: 1732301891653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:11,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:11,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:11,663 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:11,663 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:11,663 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:11,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:11,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732301891735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:11,744 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to d79ba0c344fb,43593,1732301724173 for 'test_row_1' in 2c052b4ec10710079fa7898a974e92e1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] ... 32 more 2024-11-22T18:57:11,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:11,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:11,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:11,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:11,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41726 deadline: 1732301891766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:11,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:11,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732301891766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:11,770 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to d79ba0c344fb,43593,1732301724173 for 'test_row_2' in 2c052b4ec10710079fa7898a974e92e1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] ... 32 more 2024-11-22T18:57:11,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:11,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:11,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:11,771 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:11,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732301891766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:11,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:11,771 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:11,771 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:11,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:11,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:11,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:11,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41730 deadline: 1732301891766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:11,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:11,771 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:11,771 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:11,771 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:11,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:57:11,904 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411226055c7d005fd45a4a692601c03f3a48e_2c052b4ec10710079fa7898a974e92e1 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411226055c7d005fd45a4a692601c03f3a48e_2c052b4ec10710079fa7898a974e92e1 2024-11-22T18:57:11,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/A/1740d866427c4fa38c3bbeb8b49a1f2d, store: [table=TestAcidGuarantees family=A region=2c052b4ec10710079fa7898a974e92e1] 2024-11-22T18:57:11,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/A/1740d866427c4fa38c3bbeb8b49a1f2d is 175, key is test_row_0/A:col10/1732301829583/Put/seqid=0 2024-11-22T18:57:11,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742189_1365 (size=31105) 2024-11-22T18:57:11,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-22T18:57:11,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:11,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732301891976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:11,980 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:11,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:11,980 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:11,980 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:11,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732301891976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:11,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:11,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:11,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:11,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41730 deadline: 1732301891976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:11,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:11,981 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:11,981 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:11,981 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:12,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:12,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732301892285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:12,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:12,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:12,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:12,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:12,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732301892285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:12,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:12,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:12,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41730 deadline: 1732301892285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:12,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:12,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:12,290 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:12,290 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:12,290 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:12,312 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=175, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/A/1740d866427c4fa38c3bbeb8b49a1f2d 2024-11-22T18:57:12,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/B/e3c12b04bcb44bbdb514524ee703f68b is 50, key is test_row_0/B:col10/1732301829583/Put/seqid=0 2024-11-22T18:57:12,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742190_1366 (size=12151) 2024-11-22T18:57:12,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-22T18:57:12,729 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/B/e3c12b04bcb44bbdb514524ee703f68b 2024-11-22T18:57:12,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/C/68e3734af5a34ed1b1a0fc1f7c31e9f0 is 50, key is test_row_0/C:col10/1732301829583/Put/seqid=0 2024-11-22T18:57:12,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742191_1367 (size=12151) 2024-11-22T18:57:12,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:12,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732301892795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:12,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:12,799 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:12,799 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:12,799 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:12,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732301892795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:12,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:12,800 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:12,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41730 deadline: 1732301892795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:12,800 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:12,800 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:12,800 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:12,800 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:12,800 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:13,141 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/C/68e3734af5a34ed1b1a0fc1f7c31e9f0 2024-11-22T18:57:13,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/A/1740d866427c4fa38c3bbeb8b49a1f2d as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/1740d866427c4fa38c3bbeb8b49a1f2d 2024-11-22T18:57:13,150 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/1740d866427c4fa38c3bbeb8b49a1f2d, entries=150, sequenceid=175, filesize=30.4 K 2024-11-22T18:57:13,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/B/e3c12b04bcb44bbdb514524ee703f68b as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/e3c12b04bcb44bbdb514524ee703f68b 2024-11-22T18:57:13,155 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/e3c12b04bcb44bbdb514524ee703f68b, entries=150, sequenceid=175, filesize=11.9 K 2024-11-22T18:57:13,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/C/68e3734af5a34ed1b1a0fc1f7c31e9f0 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/68e3734af5a34ed1b1a0fc1f7c31e9f0 2024-11-22T18:57:13,159 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/68e3734af5a34ed1b1a0fc1f7c31e9f0, entries=150, sequenceid=175, filesize=11.9 K 2024-11-22T18:57:13,160 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 2c052b4ec10710079fa7898a974e92e1 in 1678ms, sequenceid=175, compaction requested=true 2024-11-22T18:57:13,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for 2c052b4ec10710079fa7898a974e92e1: 2024-11-22T18:57:13,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:57:13,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-22T18:57:13,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-22T18:57:13,163 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-22T18:57:13,163 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8320 sec 2024-11-22T18:57:13,164 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 1.8370 sec 2024-11-22T18:57:13,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-22T18:57:13,475 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T18:57:13,476 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T18:57:13,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-22T18:57:13,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-22T18:57:13,478 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T18:57:13,479 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T18:57:13,479 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T18:57:13,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-22T18:57:13,631 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-22T18:57:13,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:57:13,632 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing 2c052b4ec10710079fa7898a974e92e1 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-22T18:57:13,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122ea3959124c2945ec8aded515d23a0358_2c052b4ec10710079fa7898a974e92e1 is 50, key is test_row_0/A:col10/1732301831651/Put/seqid=0 2024-11-22T18:57:13,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742192_1368 (size=12304) 2024-11-22T18:57:13,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on 2c052b4ec10710079fa7898a974e92e1 2024-11-22T18:57:13,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. as already flushing 2024-11-22T18:57:13,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-22T18:57:13,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:13,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732301893815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:13,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:13,822 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:13,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732301893815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:13,822 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:13,822 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:13,822 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to d79ba0c344fb,43593,1732301724173 for 'test_row_1' in 2c052b4ec10710079fa7898a974e92e1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] ... 32 more 2024-11-22T18:57:13,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:13,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:13,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:13,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:13,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41730 deadline: 1732301893816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:13,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:13,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41726 deadline: 1732301893817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:13,830 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:13,830 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:13,830 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:13,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:13,830 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:13,831 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:13,831 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:13,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732301893819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:13,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:13,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:13,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:13,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:13,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732301893925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:13,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:13,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:13,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:13,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:13,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41730 deadline: 1732301893935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:13,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:13,945 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:13,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41726 deadline: 1732301893935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:13,945 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:13,946 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:13,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:13,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:13,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732301893936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:13,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:13,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:13,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:13,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:13,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:14,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:57:14,052 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122ea3959124c2945ec8aded515d23a0358_2c052b4ec10710079fa7898a974e92e1 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122ea3959124c2945ec8aded515d23a0358_2c052b4ec10710079fa7898a974e92e1 2024-11-22T18:57:14,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/A/4cfe0c789bf34e0c9cb3985fbbf42872, store: [table=TestAcidGuarantees family=A region=2c052b4ec10710079fa7898a974e92e1] 2024-11-22T18:57:14,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/A/4cfe0c789bf34e0c9cb3985fbbf42872 is 175, key is test_row_0/A:col10/1732301831651/Put/seqid=0 2024-11-22T18:57:14,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742193_1369 (size=31105) 2024-11-22T18:57:14,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-22T18:57:14,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:14,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732301894135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:14,143 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:14,143 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:14,143 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:14,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:14,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41730 deadline: 1732301894156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:14,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:14,165 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:14,165 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:14,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41726 deadline: 1732301894156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:14,165 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:14,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:14,165 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:14,165 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:14,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732301894156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:14,165 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:14,166 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:14,166 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:14,166 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:14,458 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=195, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/A/4cfe0c789bf34e0c9cb3985fbbf42872 2024-11-22T18:57:14,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/B/85e3429be6ca4c3096cc34c75cb4ca68 is 50, key is test_row_0/B:col10/1732301831651/Put/seqid=0 2024-11-22T18:57:14,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:14,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732301894455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:14,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:14,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:14,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:14,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742194_1370 (size=12151) 2024-11-22T18:57:14,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:14,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41730 deadline: 1732301894475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:14,483 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:14,483 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:14,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:14,484 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:14,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41726 deadline: 1732301894475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:14,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:14,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:14,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:14,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:14,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732301894475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:14,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:14,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:14,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:14,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-22T18:57:14,870 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/B/85e3429be6ca4c3096cc34c75cb4ca68 2024-11-22T18:57:14,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/C/433504115a144e89a8ca391da06aff93 is 50, key is test_row_0/C:col10/1732301831651/Put/seqid=0 2024-11-22T18:57:14,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742195_1371 (size=12151) 2024-11-22T18:57:14,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:14,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732301894975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:14,984 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:14,984 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:14,984 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:15,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:15,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41730 deadline: 1732301894995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:15,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:15,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41726 deadline: 1732301894995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:15,004 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:15,004 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:15,004 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:15,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:15,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732301894996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:15,004 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:15,004 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:15,004 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:15,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:15,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:15,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:15,282 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/C/433504115a144e89a8ca391da06aff93 2024-11-22T18:57:15,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/A/4cfe0c789bf34e0c9cb3985fbbf42872 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/4cfe0c789bf34e0c9cb3985fbbf42872 2024-11-22T18:57:15,290 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/4cfe0c789bf34e0c9cb3985fbbf42872, entries=150, sequenceid=195, filesize=30.4 K 2024-11-22T18:57:15,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/B/85e3429be6ca4c3096cc34c75cb4ca68 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/85e3429be6ca4c3096cc34c75cb4ca68 2024-11-22T18:57:15,294 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/85e3429be6ca4c3096cc34c75cb4ca68, entries=150, sequenceid=195, filesize=11.9 K 2024-11-22T18:57:15,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/C/433504115a144e89a8ca391da06aff93 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/433504115a144e89a8ca391da06aff93 2024-11-22T18:57:15,298 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/433504115a144e89a8ca391da06aff93, entries=150, sequenceid=195, filesize=11.9 K 2024-11-22T18:57:15,299 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 2c052b4ec10710079fa7898a974e92e1 in 1668ms, sequenceid=195, compaction requested=true 2024-11-22T18:57:15,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for 2c052b4ec10710079fa7898a974e92e1: 2024-11-22T18:57:15,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:57:15,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-22T18:57:15,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-22T18:57:15,302 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-22T18:57:15,302 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8210 sec 2024-11-22T18:57:15,304 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 1.8260 sec 2024-11-22T18:57:15,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-22T18:57:15,626 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T18:57:15,627 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T18:57:15,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-22T18:57:15,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-22T18:57:15,629 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T18:57:15,630 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T18:57:15,630 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T18:57:15,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-22T18:57:15,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-22T18:57:15,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:57:15,782 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 2c052b4ec10710079fa7898a974e92e1 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-22T18:57:15,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411228a567b906f5f4916843411121aebb2b6_2c052b4ec10710079fa7898a974e92e1 is 50, key is test_row_0/A:col10/1732301833816/Put/seqid=0 2024-11-22T18:57:15,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742196_1372 (size=12304) 2024-11-22T18:57:15,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on 2c052b4ec10710079fa7898a974e92e1 2024-11-22T18:57:15,843 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. as already flushing 2024-11-22T18:57:15,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-22T18:57:16,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:16,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732301895992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:16,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:16,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732301895996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:16,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:16,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:16,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:16,007 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to d79ba0c344fb,43593,1732301724173 for 'test_row_0' in 2c052b4ec10710079fa7898a974e92e1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] ... 32 more 2024-11-22T18:57:16,008 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:16,008 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:16,008 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:16,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:16,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41730 deadline: 1732301896015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:16,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:16,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41726 deadline: 1732301896015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:16,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:16,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732301896015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:16,022 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to d79ba0c344fb,43593,1732301724173 for 'test_row_1' in 2c052b4ec10710079fa7898a974e92e1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] ... 32 more 2024-11-22T18:57:16,022 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:16,022 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:16,022 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:16,022 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to d79ba0c344fb,43593,1732301724173 for 'test_row_1' in 2c052b4ec10710079fa7898a974e92e1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] ... 32 more 2024-11-22T18:57:16,022 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:16,022 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to d79ba0c344fb,43593,1732301724173 for 'test_row_0' in 2c052b4ec10710079fa7898a974e92e1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] ... 32 more 2024-11-22T18:57:16,022 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:16,023 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:16,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:16,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:16,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:16,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:16,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732301896115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:16,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:16,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:16,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:16,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:57:16,197 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411228a567b906f5f4916843411121aebb2b6_2c052b4ec10710079fa7898a974e92e1 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411228a567b906f5f4916843411121aebb2b6_2c052b4ec10710079fa7898a974e92e1 2024-11-22T18:57:16,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/A/8559d932ad564cfda8272b07700b61b1, store: [table=TestAcidGuarantees family=A region=2c052b4ec10710079fa7898a974e92e1] 2024-11-22T18:57:16,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/A/8559d932ad564cfda8272b07700b61b1 is 175, key is test_row_0/A:col10/1732301833816/Put/seqid=0 2024-11-22T18:57:16,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742197_1373 (size=31105) 2024-11-22T18:57:16,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-22T18:57:16,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:16,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732301896336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:16,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:16,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:16,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:16,603 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=211, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/A/8559d932ad564cfda8272b07700b61b1 2024-11-22T18:57:16,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/B/81bf224fddfe420690ebdfec5eef199d is 50, key is test_row_0/B:col10/1732301833816/Put/seqid=0 2024-11-22T18:57:16,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742198_1374 (size=12151) 2024-11-22T18:57:16,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:16,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732301896655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:16,661 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:16,661 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:16,661 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:16,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-22T18:57:17,014 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/B/81bf224fddfe420690ebdfec5eef199d 2024-11-22T18:57:17,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/C/70f705b032244624b8950769c433d0b5 is 50, key is test_row_0/C:col10/1732301833816/Put/seqid=0 2024-11-22T18:57:17,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742199_1375 (size=12151) 2024-11-22T18:57:17,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:17,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732301897166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:17,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:17,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:17,178 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:17,426 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/C/70f705b032244624b8950769c433d0b5 2024-11-22T18:57:17,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/A/8559d932ad564cfda8272b07700b61b1 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/8559d932ad564cfda8272b07700b61b1 2024-11-22T18:57:17,434 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/8559d932ad564cfda8272b07700b61b1, entries=150, sequenceid=211, filesize=30.4 K 2024-11-22T18:57:17,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/B/81bf224fddfe420690ebdfec5eef199d as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/81bf224fddfe420690ebdfec5eef199d 2024-11-22T18:57:17,438 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/81bf224fddfe420690ebdfec5eef199d, entries=150, sequenceid=211, filesize=11.9 K 2024-11-22T18:57:17,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/C/70f705b032244624b8950769c433d0b5 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/70f705b032244624b8950769c433d0b5 2024-11-22T18:57:17,441 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/70f705b032244624b8950769c433d0b5, entries=150, sequenceid=211, filesize=11.9 K 2024-11-22T18:57:17,442 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 2c052b4ec10710079fa7898a974e92e1 in 1660ms, sequenceid=211, compaction requested=true 2024-11-22T18:57:17,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 2c052b4ec10710079fa7898a974e92e1: 2024-11-22T18:57:17,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:57:17,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-22T18:57:17,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-22T18:57:17,445 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-22T18:57:17,445 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8140 sec 2024-11-22T18:57:17,447 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 1.8190 sec 2024-11-22T18:57:17,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-22T18:57:17,775 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T18:57:17,776 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T18:57:17,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-22T18:57:17,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-22T18:57:17,777 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T18:57:17,778 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T18:57:17,778 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T18:57:17,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-22T18:57:17,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-22T18:57:17,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:57:17,931 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 2c052b4ec10710079fa7898a974e92e1 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-22T18:57:17,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411229d3db09799184c1bb90d0a01606a3a1c_2c052b4ec10710079fa7898a974e92e1 is 50, key is test_row_0/A:col10/1732301835991/Put/seqid=0 2024-11-22T18:57:17,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742200_1376 (size=12304) 2024-11-22T18:57:18,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on 2c052b4ec10710079fa7898a974e92e1 2024-11-22T18:57:18,032 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. as already flushing 2024-11-22T18:57:18,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:18,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732301898062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:18,073 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:18,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:18,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:18,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:18,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41730 deadline: 1732301898063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:18,077 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:18,077 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:18,077 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:18,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:18,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41726 deadline: 1732301898072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:18,082 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:18,082 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:18,082 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:18,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:18,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732301898073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:18,087 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:18,087 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:18,087 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:18,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-22T18:57:18,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:18,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732301898176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:18,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:18,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:18,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:18,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:18,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732301898186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:18,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:18,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41730 deadline: 1732301898186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:18,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:18,197 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:18,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41726 deadline: 1732301898186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:18,197 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:18,197 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:18,197 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to d79ba0c344fb,43593,1732301724173 for 'test_row_0' in 2c052b4ec10710079fa7898a974e92e1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] ... 32 more 2024-11-22T18:57:18,197 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:18,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:18,197 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:18,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:18,197 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:18,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:18,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:18,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732301898195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:18,203 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:18,203 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:18,203 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:18,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:57:18,350 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411229d3db09799184c1bb90d0a01606a3a1c_2c052b4ec10710079fa7898a974e92e1 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411229d3db09799184c1bb90d0a01606a3a1c_2c052b4ec10710079fa7898a974e92e1 2024-11-22T18:57:18,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/A/2c95925334764e74b0670552d1ee024a, store: [table=TestAcidGuarantees family=A region=2c052b4ec10710079fa7898a974e92e1] 2024-11-22T18:57:18,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/A/2c95925334764e74b0670552d1ee024a is 175, key is test_row_0/A:col10/1732301835991/Put/seqid=0 2024-11-22T18:57:18,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742201_1377 (size=31105) 2024-11-22T18:57:18,401 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:18,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732301898396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:18,402 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:18,402 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:18,402 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:18,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-22T18:57:18,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:18,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41730 deadline: 1732301898406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:18,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:18,417 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:18,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41726 deadline: 1732301898406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:18,417 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:18,417 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:18,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:18,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732301898406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:18,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:18,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:18,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:18,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:18,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:18,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:18,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:18,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732301898705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:18,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:18,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:18,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:18,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:18,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41730 deadline: 1732301898725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:18,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:18,735 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:18,735 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:18,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41726 deadline: 1732301898726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:18,735 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:18,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:18,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:18,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732301898726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:18,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:18,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:18,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:18,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:18,736 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:18,757 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=231, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/A/2c95925334764e74b0670552d1ee024a 2024-11-22T18:57:18,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/B/e18ee70debb74cf58430cca64f4bf2f4 is 50, key is test_row_0/B:col10/1732301835991/Put/seqid=0 2024-11-22T18:57:18,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742202_1378 (size=12151) 2024-11-22T18:57:18,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-22T18:57:19,169 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/B/e18ee70debb74cf58430cca64f4bf2f4 2024-11-22T18:57:19,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/C/be22a1cb379046349d969799a79e35fd is 50, key is test_row_0/C:col10/1732301835991/Put/seqid=0 2024-11-22T18:57:19,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742203_1379 (size=12151) 2024-11-22T18:57:19,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:19,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732301899225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:19,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:19,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:19,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:19,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:19,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41730 deadline: 1732301899245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:19,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:19,257 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:19,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41726 deadline: 1732301899246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:19,257 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:19,257 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:19,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:19,257 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:19,257 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:19,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732301899246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:19,257 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:19,257 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:19,257 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:19,257 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:19,580 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/C/be22a1cb379046349d969799a79e35fd 2024-11-22T18:57:19,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/A/2c95925334764e74b0670552d1ee024a as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/2c95925334764e74b0670552d1ee024a 2024-11-22T18:57:19,589 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/2c95925334764e74b0670552d1ee024a, entries=150, sequenceid=231, filesize=30.4 K 2024-11-22T18:57:19,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/B/e18ee70debb74cf58430cca64f4bf2f4 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/e18ee70debb74cf58430cca64f4bf2f4 2024-11-22T18:57:19,593 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/e18ee70debb74cf58430cca64f4bf2f4, entries=150, sequenceid=231, filesize=11.9 K 2024-11-22T18:57:19,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/C/be22a1cb379046349d969799a79e35fd as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/be22a1cb379046349d969799a79e35fd 2024-11-22T18:57:19,597 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/be22a1cb379046349d969799a79e35fd, entries=150, sequenceid=231, filesize=11.9 K 2024-11-22T18:57:19,598 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 2c052b4ec10710079fa7898a974e92e1 in 1667ms, sequenceid=231, compaction requested=true 2024-11-22T18:57:19,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 2c052b4ec10710079fa7898a974e92e1: 2024-11-22T18:57:19,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:57:19,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-22T18:57:19,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-22T18:57:19,602 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-22T18:57:19,602 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8220 sec 2024-11-22T18:57:19,603 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.8260 sec 2024-11-22T18:57:19,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-22T18:57:19,926 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T18:57:19,926 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T18:57:19,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-22T18:57:19,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-22T18:57:19,928 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T18:57:19,929 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T18:57:19,929 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T18:57:20,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-22T18:57:20,081 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-22T18:57:20,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:57:20,081 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 2c052b4ec10710079fa7898a974e92e1 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-22T18:57:20,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122b9fd1f3a1f4a4687a7a9393df30a9064_2c052b4ec10710079fa7898a974e92e1 is 50, key is test_row_0/A:col10/1732301838054/Put/seqid=0 2024-11-22T18:57:20,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742204_1380 (size=12304) 2024-11-22T18:57:20,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on 2c052b4ec10710079fa7898a974e92e1 2024-11-22T18:57:20,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. as already flushing 2024-11-22T18:57:20,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-22T18:57:20,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:20,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732301900283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:20,302 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:20,302 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:20,302 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:20,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:20,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41730 deadline: 1732301900289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:20,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:20,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:20,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:20,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41726 deadline: 1732301900298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:20,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:20,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:20,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:20,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:20,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:20,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732301900299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:20,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:20,314 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:20,314 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:20,314 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:20,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732301900300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:20,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:20,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:20,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:20,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:20,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732301900406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:20,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:20,418 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:20,418 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:20,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41730 deadline: 1732301900406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:20,418 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:20,418 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:20,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:20,418 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:20,418 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:20,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41726 deadline: 1732301900406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:20,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:20,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:20,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:20,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:20,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732301900425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:20,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:20,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:20,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:20,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:20,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732301900425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:20,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:20,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:20,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:20,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:57:20,497 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122b9fd1f3a1f4a4687a7a9393df30a9064_2c052b4ec10710079fa7898a974e92e1 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122b9fd1f3a1f4a4687a7a9393df30a9064_2c052b4ec10710079fa7898a974e92e1 2024-11-22T18:57:20,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/A/403ac6d1007346258b5ddc68610e45c2, store: [table=TestAcidGuarantees family=A region=2c052b4ec10710079fa7898a974e92e1] 2024-11-22T18:57:20,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/A/403ac6d1007346258b5ddc68610e45c2 is 175, key is test_row_0/A:col10/1732301838054/Put/seqid=0 2024-11-22T18:57:20,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742205_1381 (size=31105) 2024-11-22T18:57:20,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-22T18:57:20,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:20,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41730 deadline: 1732301900626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:20,644 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:20,644 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:20,644 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:20,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:20,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732301900626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:20,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:20,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:20,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:20,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:20,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41726 deadline: 1732301900626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:20,645 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:20,645 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:20,645 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:20,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:20,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732301900645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:20,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:20,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:20,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:20,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732301900645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:20,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:20,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:20,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:20,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:20,847 INFO [Thread-1479 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1479. 2024-11-22T18:57:20,847 DEBUG [Thread-1479 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T18:57:20,847 DEBUG [Thread-1479 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:57:20,847 DEBUG [Thread-1479 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:57:20,847 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T18:57:20,848 INFO [Thread-1477 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1477. 2024-11-22T18:57:20,848 DEBUG [Thread-1477 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T18:57:20,848 DEBUG [Thread-1477 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:57:20,848 DEBUG [Thread-1477 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:57:20,848 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T18:57:20,855 INFO [Thread-1471 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1471. 2024-11-22T18:57:20,855 DEBUG [Thread-1471 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T18:57:20,855 DEBUG [Thread-1471 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:57:20,856 DEBUG [Thread-1471 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:57:20,856 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T18:57:20,861 INFO [Thread-1475 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1475. 2024-11-22T18:57:20,861 DEBUG [Thread-1475 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T18:57:20,861 DEBUG [Thread-1475 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:57:20,861 DEBUG [Thread-1475 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:57:20,861 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T18:57:20,861 INFO [Thread-1473 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1473. 2024-11-22T18:57:20,862 DEBUG [Thread-1473 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T18:57:20,862 DEBUG [Thread-1473 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:57:20,862 DEBUG [Thread-1473 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:57:20,862 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T18:57:20,903 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=247, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/A/403ac6d1007346258b5ddc68610e45c2 2024-11-22T18:57:20,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/B/9d9b62ce0e3d41b39752f07e704741db is 50, key is test_row_0/B:col10/1732301838054/Put/seqid=0 2024-11-22T18:57:20,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742206_1382 (size=12151) 2024-11-22T18:57:20,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:20,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:20,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:20,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41730 deadline: 1732301900956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:20,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732301900956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:20,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41726 deadline: 1732301900956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:20,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:20,956 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:20,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732301900956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:20,956 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:20,956 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:20,956 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:20,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:20,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:20,956 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:20,956 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:20,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:20,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:20,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:20,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:20,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:20,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732301900965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:20,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:20,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:20,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:21,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-22T18:57:21,312 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/B/9d9b62ce0e3d41b39752f07e704741db 2024-11-22T18:57:21,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/C/76b428948c9842919561ffcb859153f9 is 50, key is test_row_0/C:col10/1732301838054/Put/seqid=0 2024-11-22T18:57:21,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742207_1383 (size=12151) 2024-11-22T18:57:21,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:21,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:21,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:21,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41730 deadline: 1732301901465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:21,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41718 deadline: 1732301901465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:21,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41726 deadline: 1732301901465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:21,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:21,466 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:21,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732301901465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:21,466 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:21,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:21,466 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:21,466 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:21,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:21,466 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:21,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:21,466 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:21,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:21,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:21,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:21,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:21,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41708 deadline: 1732301901476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:21,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:21,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:21,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:57:21,721 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/C/76b428948c9842919561ffcb859153f9 2024-11-22T18:57:21,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/A/403ac6d1007346258b5ddc68610e45c2 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/403ac6d1007346258b5ddc68610e45c2 2024-11-22T18:57:21,728 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/403ac6d1007346258b5ddc68610e45c2, entries=150, sequenceid=247, filesize=30.4 K 2024-11-22T18:57:21,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/B/9d9b62ce0e3d41b39752f07e704741db as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/9d9b62ce0e3d41b39752f07e704741db 2024-11-22T18:57:21,731 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/9d9b62ce0e3d41b39752f07e704741db, entries=150, sequenceid=247, filesize=11.9 K 2024-11-22T18:57:21,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/C/76b428948c9842919561ffcb859153f9 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/76b428948c9842919561ffcb859153f9 2024-11-22T18:57:21,734 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/76b428948c9842919561ffcb859153f9, entries=150, sequenceid=247, filesize=11.9 K 2024-11-22T18:57:21,735 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 2c052b4ec10710079fa7898a974e92e1 in 1654ms, sequenceid=247, compaction requested=true 2024-11-22T18:57:21,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 2c052b4ec10710079fa7898a974e92e1: 2024-11-22T18:57:21,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:57:21,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-22T18:57:21,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-22T18:57:21,737 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-22T18:57:21,737 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8070 sec 2024-11-22T18:57:21,738 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.8120 sec 2024-11-22T18:57:22,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-22T18:57:22,076 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T18:57:22,429 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-22T18:57:22,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on 2c052b4ec10710079fa7898a974e92e1 2024-11-22T18:57:22,477 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T18:57:22,477 INFO [Thread-1460 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1460. 2024-11-22T18:57:22,477 DEBUG [Thread-1460 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T18:57:22,477 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2c052b4ec10710079fa7898a974e92e1 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-22T18:57:22,477 DEBUG [Thread-1460 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:57:22,477 INFO [Thread-1462 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1462. 2024-11-22T18:57:22,477 DEBUG [Thread-1462 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T18:57:22,477 DEBUG [Thread-1462 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:57:22,477 DEBUG [Thread-1460 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:57:22,477 DEBUG [Thread-1462 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:57:22,477 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T18:57:22,477 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T18:57:22,482 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122d0fdf29cb6ce4cce8a3e1b9aa1c1bef2_2c052b4ec10710079fa7898a974e92e1 is 50, key is test_row_0/A:col10/1732301842475/Put/seqid=0 2024-11-22T18:57:22,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742208_1384 (size=12454) 2024-11-22T18:57:22,486 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:57:22,487 INFO [Thread-1466 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1466. 2024-11-22T18:57:22,487 DEBUG [Thread-1466 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T18:57:22,487 DEBUG [Thread-1466 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:57:22,487 DEBUG [Thread-1466 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:57:22,487 INFO [Thread-1468 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1468. 2024-11-22T18:57:22,487 DEBUG [Thread-1468 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T18:57:22,487 DEBUG [Thread-1468 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:57:22,487 INFO [Thread-1464 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1464. 2024-11-22T18:57:22,487 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T18:57:22,487 DEBUG [Thread-1468 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:57:22,487 DEBUG [Thread-1464 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T18:57:22,487 DEBUG [Thread-1464 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:57:22,487 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T18:57:22,487 DEBUG [Thread-1464 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:57:22,487 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T18:57:22,488 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-22T18:57:22,488 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 34 2024-11-22T18:57:22,488 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 46 2024-11-22T18:57:22,488 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 43 2024-11-22T18:57:22,488 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 40 2024-11-22T18:57:22,488 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-11-22T18:57:22,488 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-22T18:57:22,488 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-22T18:57:22,488 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1181 2024-11-22T18:57:22,488 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3540 rows 2024-11-22T18:57:22,488 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1179 2024-11-22T18:57:22,488 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3534 rows 2024-11-22T18:57:22,488 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1188 2024-11-22T18:57:22,488 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3558 rows 2024-11-22T18:57:22,488 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1187 2024-11-22T18:57:22,488 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3558 rows 2024-11-22T18:57:22,488 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1178 2024-11-22T18:57:22,488 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3534 rows 2024-11-22T18:57:22,488 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-22T18:57:22,488 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T18:57:22,488 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:57:22,488 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:57:22,488 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T18:57:22,489 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-22T18:57:22,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-22T18:57:22,490 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122d0fdf29cb6ce4cce8a3e1b9aa1c1bef2_2c052b4ec10710079fa7898a974e92e1 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122d0fdf29cb6ce4cce8a3e1b9aa1c1bef2_2c052b4ec10710079fa7898a974e92e1 2024-11-22T18:57:22,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-22T18:57:22,491 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/A/1bc8139dee1946ed8f4822c1ddfc7a3b, store: [table=TestAcidGuarantees family=A region=2c052b4ec10710079fa7898a974e92e1] 2024-11-22T18:57:22,492 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/A/1bc8139dee1946ed8f4822c1ddfc7a3b is 175, key is test_row_0/A:col10/1732301842475/Put/seqid=0 2024-11-22T18:57:22,492 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732301842492"}]},"ts":"1732301842492"} 2024-11-22T18:57:22,493 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-22T18:57:22,493 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-22T18:57:22,493 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-22T18:57:22,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742209_1385 (size=31255) 2024-11-22T18:57:22,495 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2c052b4ec10710079fa7898a974e92e1, UNASSIGN}] 2024-11-22T18:57:22,496 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2c052b4ec10710079fa7898a974e92e1, UNASSIGN 2024-11-22T18:57:22,497 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=125 updating hbase:meta row=2c052b4ec10710079fa7898a974e92e1, regionState=CLOSING, regionLocation=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:22,498 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2c052b4ec10710079fa7898a974e92e1, UNASSIGN because future has completed 2024-11-22T18:57:22,498 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-22T18:57:22,498 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173}] 2024-11-22T18:57:22,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-22T18:57:22,650 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] handler.UnassignRegionHandler(122): Close 2c052b4ec10710079fa7898a974e92e1 2024-11-22T18:57:22,650 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-22T18:57:22,650 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(1722): Closing 2c052b4ec10710079fa7898a974e92e1, disabling compactions & flushes 2024-11-22T18:57:22,650 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:57:22,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-22T18:57:22,896 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=269, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/A/1bc8139dee1946ed8f4822c1ddfc7a3b 2024-11-22T18:57:22,901 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/B/c4fc549f18b84f9f83b35653f473f797 is 50, key is test_row_0/B:col10/1732301842475/Put/seqid=0 2024-11-22T18:57:22,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742210_1386 (size=12301) 2024-11-22T18:57:23,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-22T18:57:23,305 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/B/c4fc549f18b84f9f83b35653f473f797 2024-11-22T18:57:23,311 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/C/8f9d21bfc094495e8d647b43189091b7 is 50, key is test_row_0/C:col10/1732301842475/Put/seqid=0 2024-11-22T18:57:23,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742211_1387 (size=12301) 2024-11-22T18:57:23,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-22T18:57:23,715 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/C/8f9d21bfc094495e8d647b43189091b7 2024-11-22T18:57:23,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/A/1bc8139dee1946ed8f4822c1ddfc7a3b as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/1bc8139dee1946ed8f4822c1ddfc7a3b 2024-11-22T18:57:23,722 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/1bc8139dee1946ed8f4822c1ddfc7a3b, entries=150, sequenceid=269, filesize=30.5 K 2024-11-22T18:57:23,722 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/B/c4fc549f18b84f9f83b35653f473f797 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/c4fc549f18b84f9f83b35653f473f797 2024-11-22T18:57:23,725 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/c4fc549f18b84f9f83b35653f473f797, entries=150, sequenceid=269, filesize=12.0 K 2024-11-22T18:57:23,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/C/8f9d21bfc094495e8d647b43189091b7 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/8f9d21bfc094495e8d647b43189091b7 2024-11-22T18:57:23,728 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/8f9d21bfc094495e8d647b43189091b7, entries=150, sequenceid=269, filesize=12.0 K 2024-11-22T18:57:23,729 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=20.13 KB/20610 for 2c052b4ec10710079fa7898a974e92e1 in 1251ms, sequenceid=269, compaction requested=true 2024-11-22T18:57:23,729 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2c052b4ec10710079fa7898a974e92e1: 2024-11-22T18:57:23,729 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:57:23,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2c052b4ec10710079fa7898a974e92e1:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T18:57:23,729 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:57:23,729 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. because compaction request was cancelled 2024-11-22T18:57:23,729 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. after waiting 0 ms 2024-11-22T18:57:23,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:57:23,729 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2c052b4ec10710079fa7898a974e92e1:A 2024-11-22T18:57:23,729 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:57:23,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2c052b4ec10710079fa7898a974e92e1:B, priority=-2147483648, current under compaction store size is 1 2024-11-22T18:57:23,729 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. because compaction request was cancelled 2024-11-22T18:57:23,729 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2c052b4ec10710079fa7898a974e92e1:B 2024-11-22T18:57:23,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:57:23,729 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(2902): Flushing 2c052b4ec10710079fa7898a974e92e1 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-22T18:57:23,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2c052b4ec10710079fa7898a974e92e1:C, priority=-2147483648, current under compaction store size is 1 2024-11-22T18:57:23,729 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. because compaction request was cancelled 2024-11-22T18:57:23,729 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2c052b4ec10710079fa7898a974e92e1:C 2024-11-22T18:57:23,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:57:23,733 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122354d1ea3d7b1433abf72532ada54926d_2c052b4ec10710079fa7898a974e92e1 is 50, key is test_row_0/A:col10/1732301842486/Put/seqid=0 2024-11-22T18:57:23,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742212_1388 (size=9914) 2024-11-22T18:57:24,137 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:57:24,140 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122354d1ea3d7b1433abf72532ada54926d_2c052b4ec10710079fa7898a974e92e1 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122354d1ea3d7b1433abf72532ada54926d_2c052b4ec10710079fa7898a974e92e1 2024-11-22T18:57:24,141 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/A/ecd875c35b3e4492ad0a09c6c005d6ba, store: [table=TestAcidGuarantees family=A region=2c052b4ec10710079fa7898a974e92e1] 2024-11-22T18:57:24,142 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/A/ecd875c35b3e4492ad0a09c6c005d6ba is 175, key is test_row_0/A:col10/1732301842486/Put/seqid=0 2024-11-22T18:57:24,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742213_1389 (size=22561) 2024-11-22T18:57:24,545 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=275, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/A/ecd875c35b3e4492ad0a09c6c005d6ba 2024-11-22T18:57:24,551 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/B/09bdfb8d7cd046048e845477970f7d7d is 50, key is test_row_0/B:col10/1732301842486/Put/seqid=0 2024-11-22T18:57:24,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742214_1390 (size=9857) 2024-11-22T18:57:24,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-22T18:57:24,955 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/B/09bdfb8d7cd046048e845477970f7d7d 2024-11-22T18:57:24,961 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/C/043d7682b6fa4df5948a44378df79b8f is 50, key is test_row_0/C:col10/1732301842486/Put/seqid=0 2024-11-22T18:57:24,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742215_1391 (size=9857) 2024-11-22T18:57:25,364 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/C/043d7682b6fa4df5948a44378df79b8f 2024-11-22T18:57:25,368 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/A/ecd875c35b3e4492ad0a09c6c005d6ba as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/ecd875c35b3e4492ad0a09c6c005d6ba 2024-11-22T18:57:25,371 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/ecd875c35b3e4492ad0a09c6c005d6ba, entries=100, sequenceid=275, filesize=22.0 K 2024-11-22T18:57:25,372 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/B/09bdfb8d7cd046048e845477970f7d7d as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/09bdfb8d7cd046048e845477970f7d7d 2024-11-22T18:57:25,375 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/09bdfb8d7cd046048e845477970f7d7d, entries=100, sequenceid=275, filesize=9.6 K 2024-11-22T18:57:25,376 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/.tmp/C/043d7682b6fa4df5948a44378df79b8f as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/043d7682b6fa4df5948a44378df79b8f 2024-11-22T18:57:25,378 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/043d7682b6fa4df5948a44378df79b8f, entries=100, sequenceid=275, filesize=9.6 K 2024-11-22T18:57:25,379 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 2c052b4ec10710079fa7898a974e92e1 in 1650ms, sequenceid=275, compaction requested=true 2024-11-22T18:57:25,380 DEBUG [StoreCloser-TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/074178ce8a1b41f5abcc83b728726f37, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/51821a818f754e169e8067a12a0c7530, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/2ec975cb39294402b430378b67f38ef6, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/bcb26716e9794f35b7ae03c3beeb6659, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/995c9296f3474981b0e84670de497e7e, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/67bb22be1ee8445bb25cb252c2a210f0, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/6eac306947ac4cc49139bc41e5b55c17, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/a1e2f7dfc56d4ed2b3156627df25e7fa, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/82c66899e3ab42caa9e66df18ba7db40] to archive 2024-11-22T18:57:25,380 DEBUG [StoreCloser-TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T18:57:25,381 DEBUG [StoreCloser-TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/074178ce8a1b41f5abcc83b728726f37 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/074178ce8a1b41f5abcc83b728726f37 2024-11-22T18:57:25,382 DEBUG [StoreCloser-TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/51821a818f754e169e8067a12a0c7530 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/51821a818f754e169e8067a12a0c7530 2024-11-22T18:57:25,383 DEBUG [StoreCloser-TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/2ec975cb39294402b430378b67f38ef6 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/2ec975cb39294402b430378b67f38ef6 2024-11-22T18:57:25,384 DEBUG [StoreCloser-TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/bcb26716e9794f35b7ae03c3beeb6659 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/bcb26716e9794f35b7ae03c3beeb6659 2024-11-22T18:57:25,384 DEBUG [StoreCloser-TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/995c9296f3474981b0e84670de497e7e to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/995c9296f3474981b0e84670de497e7e 2024-11-22T18:57:25,385 DEBUG [StoreCloser-TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/67bb22be1ee8445bb25cb252c2a210f0 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/67bb22be1ee8445bb25cb252c2a210f0 2024-11-22T18:57:25,386 DEBUG [StoreCloser-TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/6eac306947ac4cc49139bc41e5b55c17 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/6eac306947ac4cc49139bc41e5b55c17 2024-11-22T18:57:25,387 DEBUG [StoreCloser-TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/a1e2f7dfc56d4ed2b3156627df25e7fa to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/a1e2f7dfc56d4ed2b3156627df25e7fa 2024-11-22T18:57:25,388 DEBUG [StoreCloser-TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/82c66899e3ab42caa9e66df18ba7db40 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/82c66899e3ab42caa9e66df18ba7db40 2024-11-22T18:57:25,389 DEBUG [StoreCloser-TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/5fbb26f6a2a8440789922d1f4fd3694a, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/7901636f861044b4ad48563d8ef23557, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/8a2dc37bdb2d41be8f55f6eed0dec53c, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/2741c27e0aa041c3bc1ec62148a66e87, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/4fa0b94ef57e478b9b2e584235e93977, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/08bd76133ee145da8aa88a0d179038f6, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/39e585b55be7450bac954e7782b637dc, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/2f0997c1d44f4cf58a129f56a5719454, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/efb65377ffd54875b3fc2a0f942ff147] to archive 2024-11-22T18:57:25,389 DEBUG [StoreCloser-TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T18:57:25,390 DEBUG [StoreCloser-TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/5fbb26f6a2a8440789922d1f4fd3694a to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/5fbb26f6a2a8440789922d1f4fd3694a 2024-11-22T18:57:25,391 DEBUG [StoreCloser-TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/7901636f861044b4ad48563d8ef23557 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/7901636f861044b4ad48563d8ef23557 2024-11-22T18:57:25,392 DEBUG [StoreCloser-TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/8a2dc37bdb2d41be8f55f6eed0dec53c to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/8a2dc37bdb2d41be8f55f6eed0dec53c 2024-11-22T18:57:25,393 DEBUG [StoreCloser-TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/2741c27e0aa041c3bc1ec62148a66e87 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/2741c27e0aa041c3bc1ec62148a66e87 2024-11-22T18:57:25,393 DEBUG [StoreCloser-TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/4fa0b94ef57e478b9b2e584235e93977 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/4fa0b94ef57e478b9b2e584235e93977 2024-11-22T18:57:25,394 DEBUG [StoreCloser-TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/08bd76133ee145da8aa88a0d179038f6 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/08bd76133ee145da8aa88a0d179038f6 2024-11-22T18:57:25,395 DEBUG [StoreCloser-TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/39e585b55be7450bac954e7782b637dc to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/39e585b55be7450bac954e7782b637dc 2024-11-22T18:57:25,396 DEBUG [StoreCloser-TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/2f0997c1d44f4cf58a129f56a5719454 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/2f0997c1d44f4cf58a129f56a5719454 2024-11-22T18:57:25,397 DEBUG [StoreCloser-TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/efb65377ffd54875b3fc2a0f942ff147 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/efb65377ffd54875b3fc2a0f942ff147 2024-11-22T18:57:25,398 DEBUG [StoreCloser-TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/a85f58865dbb407bb7a45313653fd2d1, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/c000d0ff843d4f31b91b7f0240989c45, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/c78da3e492f64878b2195edac2649140, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/ade2eda6d7c844589b9db95f2a6a77cf, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/2670acd0422a4c408c08d27f4d318701, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/a0d68050bed74aa9a2e2634105a3b22d, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/301a8d024ce549c2b8f2c9136b43eff9, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/3dbc0d22f2a546c2b3fd162491ce08b0, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/9611cffd6f444e9198538a9aeb08285a] to archive 2024-11-22T18:57:25,398 DEBUG [StoreCloser-TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T18:57:25,399 DEBUG [StoreCloser-TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/a85f58865dbb407bb7a45313653fd2d1 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/a85f58865dbb407bb7a45313653fd2d1 2024-11-22T18:57:25,400 DEBUG [StoreCloser-TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/c000d0ff843d4f31b91b7f0240989c45 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/c000d0ff843d4f31b91b7f0240989c45 2024-11-22T18:57:25,401 DEBUG [StoreCloser-TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/c78da3e492f64878b2195edac2649140 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/c78da3e492f64878b2195edac2649140 2024-11-22T18:57:25,401 DEBUG [StoreCloser-TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/ade2eda6d7c844589b9db95f2a6a77cf to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/ade2eda6d7c844589b9db95f2a6a77cf 2024-11-22T18:57:25,402 DEBUG [StoreCloser-TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/2670acd0422a4c408c08d27f4d318701 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/2670acd0422a4c408c08d27f4d318701 2024-11-22T18:57:25,403 DEBUG [StoreCloser-TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/a0d68050bed74aa9a2e2634105a3b22d to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/a0d68050bed74aa9a2e2634105a3b22d 2024-11-22T18:57:25,404 DEBUG [StoreCloser-TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/301a8d024ce549c2b8f2c9136b43eff9 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/301a8d024ce549c2b8f2c9136b43eff9 2024-11-22T18:57:25,405 DEBUG [StoreCloser-TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/3dbc0d22f2a546c2b3fd162491ce08b0 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/3dbc0d22f2a546c2b3fd162491ce08b0 2024-11-22T18:57:25,406 DEBUG [StoreCloser-TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/9611cffd6f444e9198538a9aeb08285a to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/9611cffd6f444e9198538a9aeb08285a 2024-11-22T18:57:25,409 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/recovered.edits/278.seqid, newMaxSeqId=278, maxSeqId=4 2024-11-22T18:57:25,410 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. 2024-11-22T18:57:25,410 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(1676): Region close journal for 2c052b4ec10710079fa7898a974e92e1: Waiting for close lock at 1732301842650Running coprocessor pre-close hooks at 1732301842650Disabling compacts and flushes for region at 1732301842650Disabling writes for close at 1732301843729 (+1079 ms)Obtaining lock to block concurrent updates at 1732301843729Preparing flush snapshotting stores in 2c052b4ec10710079fa7898a974e92e1 at 1732301843729Finished memstore snapshotting TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=54720, getOffHeapSize=0, getCellsCount=450 at 1732301843729Flushing stores of TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1. at 1732301843729Flushing 2c052b4ec10710079fa7898a974e92e1/A: creating writer at 1732301843730 (+1 ms)Flushing mob file 2c052b4ec10710079fa7898a974e92e1/A: appending metadata at 1732301843733 (+3 ms)Flushing mob file 2c052b4ec10710079fa7898a974e92e1/A: closing flushed file at 1732301843733Flushing 2c052b4ec10710079fa7898a974e92e1/A: appending metadata at 1732301844141 (+408 ms)Flushing 2c052b4ec10710079fa7898a974e92e1/A: closing flushed file at 1732301844141Flushing 2c052b4ec10710079fa7898a974e92e1/B: creating writer at 1732301844549 (+408 ms)Flushing 2c052b4ec10710079fa7898a974e92e1/B: appending metadata at 1732301844551 (+2 ms)Flushing 2c052b4ec10710079fa7898a974e92e1/B: closing flushed file at 1732301844551Flushing 2c052b4ec10710079fa7898a974e92e1/C: creating writer at 1732301844959 (+408 ms)Flushing 2c052b4ec10710079fa7898a974e92e1/C: appending metadata at 1732301844961 (+2 ms)Flushing 2c052b4ec10710079fa7898a974e92e1/C: closing flushed file at 1732301844961Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@240a34e: reopening flushed file at 1732301845368 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@ba5f63: reopening flushed file at 1732301845372 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@346d26c9: reopening flushed file at 1732301845375 (+3 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 2c052b4ec10710079fa7898a974e92e1 in 1650ms, sequenceid=275, compaction requested=true at 1732301845379 (+4 ms)Writing region close event to WAL at 1732301845407 (+28 ms)Running coprocessor post-close hooks at 1732301845410 (+3 ms)Closed at 1732301845410 2024-11-22T18:57:25,412 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] handler.UnassignRegionHandler(157): Closed 2c052b4ec10710079fa7898a974e92e1 2024-11-22T18:57:25,412 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=125 updating hbase:meta row=2c052b4ec10710079fa7898a974e92e1, regionState=CLOSED 2024-11-22T18:57:25,413 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=126, ppid=125, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 because future has completed 2024-11-22T18:57:25,415 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-22T18:57:25,415 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; CloseRegionProcedure 2c052b4ec10710079fa7898a974e92e1, server=d79ba0c344fb,43593,1732301724173 in 2.9160 sec 2024-11-22T18:57:25,416 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=125, resume processing ppid=124 2024-11-22T18:57:25,416 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, ppid=124, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2c052b4ec10710079fa7898a974e92e1, UNASSIGN in 2.9200 sec 2024-11-22T18:57:25,418 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-22T18:57:25,418 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.9240 sec 2024-11-22T18:57:25,419 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732301845419"}]},"ts":"1732301845419"} 2024-11-22T18:57:25,420 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-22T18:57:25,420 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-22T18:57:25,421 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.9310 sec 2024-11-22T18:57:26,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-22T18:57:26,656 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-22T18:57:26,657 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-22T18:57:26,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T18:57:26,658 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=127, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T18:57:26,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-22T18:57:26,659 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=127, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T18:57:26,660 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1 2024-11-22T18:57:26,662 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A, FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B, FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C, FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/recovered.edits] 2024-11-22T18:57:26,664 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/1740d866427c4fa38c3bbeb8b49a1f2d to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/1740d866427c4fa38c3bbeb8b49a1f2d 2024-11-22T18:57:26,665 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/1bc8139dee1946ed8f4822c1ddfc7a3b to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/1bc8139dee1946ed8f4822c1ddfc7a3b 2024-11-22T18:57:26,666 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/2c95925334764e74b0670552d1ee024a to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/2c95925334764e74b0670552d1ee024a 2024-11-22T18:57:26,667 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/403ac6d1007346258b5ddc68610e45c2 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/403ac6d1007346258b5ddc68610e45c2 2024-11-22T18:57:26,668 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/425593a2767d4f50ad8d502ff2438f07 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/425593a2767d4f50ad8d502ff2438f07 2024-11-22T18:57:26,669 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/4cfe0c789bf34e0c9cb3985fbbf42872 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/4cfe0c789bf34e0c9cb3985fbbf42872 2024-11-22T18:57:26,670 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/8559d932ad564cfda8272b07700b61b1 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/8559d932ad564cfda8272b07700b61b1 2024-11-22T18:57:26,671 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/c435fab400aa4a79bc7b0aa0a6cb0f2a to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/c435fab400aa4a79bc7b0aa0a6cb0f2a 2024-11-22T18:57:26,672 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/ecd875c35b3e4492ad0a09c6c005d6ba to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/A/ecd875c35b3e4492ad0a09c6c005d6ba 2024-11-22T18:57:26,675 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/09bdfb8d7cd046048e845477970f7d7d to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/09bdfb8d7cd046048e845477970f7d7d 2024-11-22T18:57:26,676 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/204abe18575d437387cd66015e375ee0 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/204abe18575d437387cd66015e375ee0 2024-11-22T18:57:26,677 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/7814d8b792274856bc929285dc1d8a3b to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/7814d8b792274856bc929285dc1d8a3b 2024-11-22T18:57:26,678 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/81bf224fddfe420690ebdfec5eef199d to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/81bf224fddfe420690ebdfec5eef199d 2024-11-22T18:57:26,679 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/85e3429be6ca4c3096cc34c75cb4ca68 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/85e3429be6ca4c3096cc34c75cb4ca68 2024-11-22T18:57:26,680 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/9d9b62ce0e3d41b39752f07e704741db to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/9d9b62ce0e3d41b39752f07e704741db 2024-11-22T18:57:26,681 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/c4fc549f18b84f9f83b35653f473f797 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/c4fc549f18b84f9f83b35653f473f797 2024-11-22T18:57:26,682 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/e18ee70debb74cf58430cca64f4bf2f4 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/e18ee70debb74cf58430cca64f4bf2f4 2024-11-22T18:57:26,683 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/e3c12b04bcb44bbdb514524ee703f68b to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/B/e3c12b04bcb44bbdb514524ee703f68b 2024-11-22T18:57:26,686 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/043d7682b6fa4df5948a44378df79b8f to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/043d7682b6fa4df5948a44378df79b8f 2024-11-22T18:57:26,687 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/433504115a144e89a8ca391da06aff93 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/433504115a144e89a8ca391da06aff93 2024-11-22T18:57:26,688 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/68e3734af5a34ed1b1a0fc1f7c31e9f0 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/68e3734af5a34ed1b1a0fc1f7c31e9f0 2024-11-22T18:57:26,689 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/70f705b032244624b8950769c433d0b5 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/70f705b032244624b8950769c433d0b5 2024-11-22T18:57:26,690 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/76b428948c9842919561ffcb859153f9 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/76b428948c9842919561ffcb859153f9 2024-11-22T18:57:26,691 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/80341b9241a244418395e20b5f1e9ffc to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/80341b9241a244418395e20b5f1e9ffc 2024-11-22T18:57:26,692 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/8f9d21bfc094495e8d647b43189091b7 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/8f9d21bfc094495e8d647b43189091b7 2024-11-22T18:57:26,693 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/baec2ea185ae4f93b4d92b96fc1e4bad to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/baec2ea185ae4f93b4d92b96fc1e4bad 2024-11-22T18:57:26,694 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/be22a1cb379046349d969799a79e35fd to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/C/be22a1cb379046349d969799a79e35fd 2024-11-22T18:57:26,696 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/recovered.edits/278.seqid to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1/recovered.edits/278.seqid 2024-11-22T18:57:26,696 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/2c052b4ec10710079fa7898a974e92e1 2024-11-22T18:57:26,697 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-22T18:57:26,697 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-22T18:57:26,698 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-22T18:57:26,701 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112210270a9fe7644aa0910be31f0c87808e_2c052b4ec10710079fa7898a974e92e1 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112210270a9fe7644aa0910be31f0c87808e_2c052b4ec10710079fa7898a974e92e1 2024-11-22T18:57:26,702 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112214e4d10b7574497599b3b43e19c424fe_2c052b4ec10710079fa7898a974e92e1 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112214e4d10b7574497599b3b43e19c424fe_2c052b4ec10710079fa7898a974e92e1 2024-11-22T18:57:26,703 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122354d1ea3d7b1433abf72532ada54926d_2c052b4ec10710079fa7898a974e92e1 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122354d1ea3d7b1433abf72532ada54926d_2c052b4ec10710079fa7898a974e92e1 2024-11-22T18:57:26,704 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411223b3ffcac0c9340a4b482949c8f06fa43_2c052b4ec10710079fa7898a974e92e1 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411223b3ffcac0c9340a4b482949c8f06fa43_2c052b4ec10710079fa7898a974e92e1 2024-11-22T18:57:26,705 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411225d3c19caa1a3435eaefc2880cfb5d5e1_2c052b4ec10710079fa7898a974e92e1 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411225d3c19caa1a3435eaefc2880cfb5d5e1_2c052b4ec10710079fa7898a974e92e1 2024-11-22T18:57:26,706 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411226055c7d005fd45a4a692601c03f3a48e_2c052b4ec10710079fa7898a974e92e1 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411226055c7d005fd45a4a692601c03f3a48e_2c052b4ec10710079fa7898a974e92e1 2024-11-22T18:57:26,708 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112264ac991c44904684bea720d28f980841_2c052b4ec10710079fa7898a974e92e1 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112264ac991c44904684bea720d28f980841_2c052b4ec10710079fa7898a974e92e1 2024-11-22T18:57:26,709 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122809f9f9e14f2462fa3887e860a011946_2c052b4ec10710079fa7898a974e92e1 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122809f9f9e14f2462fa3887e860a011946_2c052b4ec10710079fa7898a974e92e1 2024-11-22T18:57:26,710 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411228a567b906f5f4916843411121aebb2b6_2c052b4ec10710079fa7898a974e92e1 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411228a567b906f5f4916843411121aebb2b6_2c052b4ec10710079fa7898a974e92e1 2024-11-22T18:57:26,711 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112294082aa1e81e43f6b384f225060e9023_2c052b4ec10710079fa7898a974e92e1 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112294082aa1e81e43f6b384f225060e9023_2c052b4ec10710079fa7898a974e92e1 2024-11-22T18:57:26,712 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411229d3db09799184c1bb90d0a01606a3a1c_2c052b4ec10710079fa7898a974e92e1 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411229d3db09799184c1bb90d0a01606a3a1c_2c052b4ec10710079fa7898a974e92e1 2024-11-22T18:57:26,713 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122b9fd1f3a1f4a4687a7a9393df30a9064_2c052b4ec10710079fa7898a974e92e1 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122b9fd1f3a1f4a4687a7a9393df30a9064_2c052b4ec10710079fa7898a974e92e1 2024-11-22T18:57:26,714 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122d0fdf29cb6ce4cce8a3e1b9aa1c1bef2_2c052b4ec10710079fa7898a974e92e1 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122d0fdf29cb6ce4cce8a3e1b9aa1c1bef2_2c052b4ec10710079fa7898a974e92e1 2024-11-22T18:57:26,715 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122d7c5186dd0ea430c994d506caeb34115_2c052b4ec10710079fa7898a974e92e1 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122d7c5186dd0ea430c994d506caeb34115_2c052b4ec10710079fa7898a974e92e1 2024-11-22T18:57:26,716 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122ea3959124c2945ec8aded515d23a0358_2c052b4ec10710079fa7898a974e92e1 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122ea3959124c2945ec8aded515d23a0358_2c052b4ec10710079fa7898a974e92e1 2024-11-22T18:57:26,716 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-22T18:57:26,718 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=127, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T18:57:26,720 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-22T18:57:26,722 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-22T18:57:26,722 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=127, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T18:57:26,722 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-22T18:57:26,723 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732301846723"}]},"ts":"9223372036854775807"} 2024-11-22T18:57:26,724 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-22T18:57:26,724 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 2c052b4ec10710079fa7898a974e92e1, NAME => 'TestAcidGuarantees,,1732301817744.2c052b4ec10710079fa7898a974e92e1.', STARTKEY => '', ENDKEY => ''}] 2024-11-22T18:57:26,724 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-22T18:57:26,725 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732301846724"}]},"ts":"9223372036854775807"} 2024-11-22T18:57:26,726 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-22T18:57:26,726 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=127, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T18:57:26,727 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 69 msec 2024-11-22T18:57:26,748 WARN [regionserver/d79ba0c344fb:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 3, running: 1 2024-11-22T18:57:26,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-22T18:57:26,766 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-22T18:57:26,766 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-22T18:57:26,775 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=240 (was 240), OpenFileDescriptor=449 (was 455), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=304 (was 304), ProcessCount=11 (was 11), AvailableMemoryMB=7764 (was 7808) 2024-11-22T18:57:26,783 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=240, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=304, ProcessCount=11, AvailableMemoryMB=7764 2024-11-22T18:57:26,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-22T18:57:26,784 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T18:57:26,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] procedure2.ProcedureExecutor(1139): Stored pid=128, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-22T18:57:26,786 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=128, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-22T18:57:26,786 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:57:26,786 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 128 2024-11-22T18:57:26,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-11-22T18:57:26,787 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=128, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-22T18:57:26,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742216_1392 (size=959) 2024-11-22T18:57:26,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-11-22T18:57:27,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-11-22T18:57:27,193 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875 2024-11-22T18:57:27,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742217_1393 (size=53) 2024-11-22T18:57:27,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-11-22T18:57:27,599 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T18:57:27,599 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing b35207b85288b14a567277d6fef4534f, disabling compactions & flushes 2024-11-22T18:57:27,599 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:27,599 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:27,599 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. after waiting 0 ms 2024-11-22T18:57:27,599 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:27,599 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:27,599 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for b35207b85288b14a567277d6fef4534f: Waiting for close lock at 1732301847599Disabling compacts and flushes for region at 1732301847599Disabling writes for close at 1732301847599Writing region close event to WAL at 1732301847599Closed at 1732301847599 2024-11-22T18:57:27,600 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=128, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-22T18:57:27,600 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732301847600"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732301847600"}]},"ts":"1732301847600"} 2024-11-22T18:57:27,601 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-22T18:57:27,602 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=128, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-22T18:57:27,602 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732301847602"}]},"ts":"1732301847602"} 2024-11-22T18:57:27,603 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-22T18:57:27,603 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b35207b85288b14a567277d6fef4534f, ASSIGN}] 2024-11-22T18:57:27,604 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b35207b85288b14a567277d6fef4534f, ASSIGN 2024-11-22T18:57:27,605 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b35207b85288b14a567277d6fef4534f, ASSIGN; state=OFFLINE, location=d79ba0c344fb,43593,1732301724173; forceNewPlan=false, retain=false 2024-11-22T18:57:27,755 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=129 updating hbase:meta row=b35207b85288b14a567277d6fef4534f, regionState=OPENING, regionLocation=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:27,757 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b35207b85288b14a567277d6fef4534f, ASSIGN because future has completed 2024-11-22T18:57:27,757 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; OpenRegionProcedure b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173}] 2024-11-22T18:57:27,911 INFO [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:27,912 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(7752): Opening region: {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} 2024-11-22T18:57:27,912 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees b35207b85288b14a567277d6fef4534f 2024-11-22T18:57:27,912 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T18:57:27,913 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(7794): checking encryption for b35207b85288b14a567277d6fef4534f 2024-11-22T18:57:27,913 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(7797): checking classloading for b35207b85288b14a567277d6fef4534f 2024-11-22T18:57:27,914 INFO [StoreOpener-b35207b85288b14a567277d6fef4534f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region b35207b85288b14a567277d6fef4534f 2024-11-22T18:57:27,915 INFO [StoreOpener-b35207b85288b14a567277d6fef4534f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b35207b85288b14a567277d6fef4534f columnFamilyName A 2024-11-22T18:57:27,915 DEBUG [StoreOpener-b35207b85288b14a567277d6fef4534f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:57:27,916 INFO [StoreOpener-b35207b85288b14a567277d6fef4534f-1 {}] regionserver.HStore(327): Store=b35207b85288b14a567277d6fef4534f/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T18:57:27,916 INFO [StoreOpener-b35207b85288b14a567277d6fef4534f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region b35207b85288b14a567277d6fef4534f 2024-11-22T18:57:27,917 INFO [StoreOpener-b35207b85288b14a567277d6fef4534f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b35207b85288b14a567277d6fef4534f columnFamilyName B 2024-11-22T18:57:27,917 DEBUG [StoreOpener-b35207b85288b14a567277d6fef4534f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:57:27,917 INFO [StoreOpener-b35207b85288b14a567277d6fef4534f-1 {}] regionserver.HStore(327): Store=b35207b85288b14a567277d6fef4534f/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T18:57:27,917 INFO [StoreOpener-b35207b85288b14a567277d6fef4534f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region b35207b85288b14a567277d6fef4534f 2024-11-22T18:57:27,918 INFO [StoreOpener-b35207b85288b14a567277d6fef4534f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b35207b85288b14a567277d6fef4534f columnFamilyName C 2024-11-22T18:57:27,918 DEBUG [StoreOpener-b35207b85288b14a567277d6fef4534f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:57:27,918 INFO [StoreOpener-b35207b85288b14a567277d6fef4534f-1 {}] regionserver.HStore(327): Store=b35207b85288b14a567277d6fef4534f/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T18:57:27,919 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(1038): replaying wal for b35207b85288b14a567277d6fef4534f 2024-11-22T18:57:27,919 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f 2024-11-22T18:57:27,919 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f 2024-11-22T18:57:27,920 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(1048): stopping wal replay for b35207b85288b14a567277d6fef4534f 2024-11-22T18:57:27,920 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(1060): Cleaning up temporary data for b35207b85288b14a567277d6fef4534f 2024-11-22T18:57:27,921 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-22T18:57:27,922 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(1093): writing seq id for b35207b85288b14a567277d6fef4534f 2024-11-22T18:57:27,923 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T18:57:27,924 INFO [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(1114): Opened b35207b85288b14a567277d6fef4534f; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61217020, jitterRate=-0.08779531717300415}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-22T18:57:27,924 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b35207b85288b14a567277d6fef4534f 2024-11-22T18:57:27,924 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(1006): Region open journal for b35207b85288b14a567277d6fef4534f: Running coprocessor pre-open hook at 1732301847913Writing region info on filesystem at 1732301847913Initializing all the Stores at 1732301847913Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732301847913Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732301847914 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732301847914Cleaning up temporary data from old regions at 1732301847920 (+6 ms)Running coprocessor post-open hooks at 1732301847924 (+4 ms)Region opened successfully at 1732301847924 2024-11-22T18:57:27,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-11-22T18:57:27,925 INFO [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., pid=130, masterSystemTime=1732301847909 2024-11-22T18:57:27,926 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:27,926 INFO [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:27,927 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=129 updating hbase:meta row=b35207b85288b14a567277d6fef4534f, regionState=OPEN, openSeqNum=2, regionLocation=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:27,928 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=130, ppid=129, state=RUNNABLE, hasLock=false; OpenRegionProcedure b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 because future has completed 2024-11-22T18:57:27,930 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-22T18:57:27,930 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; OpenRegionProcedure b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 in 172 msec 2024-11-22T18:57:27,931 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=129, resume processing ppid=128 2024-11-22T18:57:27,931 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, ppid=128, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b35207b85288b14a567277d6fef4534f, ASSIGN in 327 msec 2024-11-22T18:57:27,932 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=128, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-22T18:57:27,932 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732301847932"}]},"ts":"1732301847932"} 2024-11-22T18:57:27,933 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-22T18:57:27,934 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=128, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-22T18:57:27,935 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1500 sec 2024-11-22T18:57:28,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-11-22T18:57:28,936 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-22T18:57:28,937 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b8abc8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:57:28,937 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d79ba0c344fb,44109,-1 for getting cluster id 2024-11-22T18:57:28,937 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T18:57:28,938 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '058c0dbe-7061-419a-804d-605a1c1b6341' 2024-11-22T18:57:28,938 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T18:57:28,938 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "058c0dbe-7061-419a-804d-605a1c1b6341" 2024-11-22T18:57:28,938 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d79017a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:57:28,938 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d79ba0c344fb,44109,-1] 2024-11-22T18:57:28,939 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T18:57:28,939 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:57:28,940 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38474, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T18:57:28,940 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f486dc1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:57:28,940 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T18:57:28,941 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d79ba0c344fb,43593,1732301724173, seqNum=-1] 2024-11-22T18:57:28,941 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T18:57:28,942 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34972, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T18:57:28,942 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-22T18:57:28,943 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is d79ba0c344fb,44109,1732301723398 2024-11-22T18:57:28,943 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@640e8a2c 2024-11-22T18:57:28,943 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-22T18:57:28,944 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38486, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-22T18:57:28,945 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26bd122f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:57:28,945 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d79ba0c344fb,44109,-1 for getting cluster id 2024-11-22T18:57:28,945 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T18:57:28,946 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '058c0dbe-7061-419a-804d-605a1c1b6341' 2024-11-22T18:57:28,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T18:57:28,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "058c0dbe-7061-419a-804d-605a1c1b6341" 2024-11-22T18:57:28,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a7da0c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:57:28,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d79ba0c344fb,44109,-1] 2024-11-22T18:57:28,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T18:57:28,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:57:28,947 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38504, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T18:57:28,947 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@af0a1e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:57:28,948 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f0c05eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:57:28,948 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d79ba0c344fb,44109,-1 for getting cluster id 2024-11-22T18:57:28,948 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T18:57:28,948 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '058c0dbe-7061-419a-804d-605a1c1b6341' 2024-11-22T18:57:28,948 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T18:57:28,949 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "058c0dbe-7061-419a-804d-605a1c1b6341" 2024-11-22T18:57:28,949 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58f5b2cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:57:28,949 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d79ba0c344fb,44109,-1] 2024-11-22T18:57:28,949 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T18:57:28,949 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:57:28,950 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38532, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T18:57:28,950 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1835ac1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:57:28,950 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79d7458, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:57:28,950 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d79ba0c344fb,44109,-1 for getting cluster id 2024-11-22T18:57:28,951 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T18:57:28,951 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '058c0dbe-7061-419a-804d-605a1c1b6341' 2024-11-22T18:57:28,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T18:57:28,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "058c0dbe-7061-419a-804d-605a1c1b6341" 2024-11-22T18:57:28,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7027e5ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:57:28,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d79ba0c344fb,44109,-1] 2024-11-22T18:57:28,952 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T18:57:28,952 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:57:28,952 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38542, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T18:57:28,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@680bdcdc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:57:28,953 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32c61cf9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:57:28,953 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d79ba0c344fb,44109,-1 for getting cluster id 2024-11-22T18:57:28,953 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T18:57:28,954 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '058c0dbe-7061-419a-804d-605a1c1b6341' 2024-11-22T18:57:28,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T18:57:28,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "058c0dbe-7061-419a-804d-605a1c1b6341" 2024-11-22T18:57:28,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54e073c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:57:28,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d79ba0c344fb,44109,-1] 2024-11-22T18:57:28,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T18:57:28,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:57:28,955 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38552, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T18:57:28,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6660b645, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:57:28,956 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c296bf5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:57:28,956 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d79ba0c344fb,44109,-1 for getting cluster id 2024-11-22T18:57:28,956 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T18:57:28,956 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '058c0dbe-7061-419a-804d-605a1c1b6341' 2024-11-22T18:57:28,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T18:57:28,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "058c0dbe-7061-419a-804d-605a1c1b6341" 2024-11-22T18:57:28,957 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67d9f2a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:57:28,957 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d79ba0c344fb,44109,-1] 2024-11-22T18:57:28,957 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T18:57:28,957 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:57:28,957 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38570, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T18:57:28,958 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3477cc17, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:57:28,958 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ff7674f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:57:28,958 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d79ba0c344fb,44109,-1 for getting cluster id 2024-11-22T18:57:28,958 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T18:57:28,959 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '058c0dbe-7061-419a-804d-605a1c1b6341' 2024-11-22T18:57:28,959 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T18:57:28,959 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "058c0dbe-7061-419a-804d-605a1c1b6341" 2024-11-22T18:57:28,959 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f46733e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:57:28,959 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d79ba0c344fb,44109,-1] 2024-11-22T18:57:28,959 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T18:57:28,959 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:57:28,960 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38584, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T18:57:28,960 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19d8b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:57:28,961 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@672fe15e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:57:28,961 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d79ba0c344fb,44109,-1 for getting cluster id 2024-11-22T18:57:28,961 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T18:57:28,961 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '058c0dbe-7061-419a-804d-605a1c1b6341' 2024-11-22T18:57:28,961 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T18:57:28,962 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "058c0dbe-7061-419a-804d-605a1c1b6341" 2024-11-22T18:57:28,962 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@279b8ad1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:57:28,962 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d79ba0c344fb,44109,-1] 2024-11-22T18:57:28,962 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T18:57:28,962 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:57:28,963 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38602, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T18:57:28,963 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3809019c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:57:28,963 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71238b4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:57:28,963 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d79ba0c344fb,44109,-1 for getting cluster id 2024-11-22T18:57:28,964 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T18:57:28,964 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '058c0dbe-7061-419a-804d-605a1c1b6341' 2024-11-22T18:57:28,964 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T18:57:28,964 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "058c0dbe-7061-419a-804d-605a1c1b6341" 2024-11-22T18:57:28,964 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55cfa4c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:57:28,965 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d79ba0c344fb,44109,-1] 2024-11-22T18:57:28,965 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T18:57:28,965 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:57:28,965 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38614, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T18:57:28,966 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56a753f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:57:28,966 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@625b319d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:57:28,966 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d79ba0c344fb,44109,-1 for getting cluster id 2024-11-22T18:57:28,966 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T18:57:28,967 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '058c0dbe-7061-419a-804d-605a1c1b6341' 2024-11-22T18:57:28,967 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T18:57:28,967 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "058c0dbe-7061-419a-804d-605a1c1b6341" 2024-11-22T18:57:28,967 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63ab8669, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:57:28,967 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d79ba0c344fb,44109,-1] 2024-11-22T18:57:28,967 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T18:57:28,967 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:57:28,968 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38644, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T18:57:28,968 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56eabd63, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:57:28,969 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49cd7b43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:57:28,969 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d79ba0c344fb,44109,-1 for getting cluster id 2024-11-22T18:57:28,969 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T18:57:28,969 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '058c0dbe-7061-419a-804d-605a1c1b6341' 2024-11-22T18:57:28,970 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T18:57:28,970 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "058c0dbe-7061-419a-804d-605a1c1b6341" 2024-11-22T18:57:28,970 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2724635, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:57:28,970 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d79ba0c344fb,44109,-1] 2024-11-22T18:57:28,970 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T18:57:28,970 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:57:28,970 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38662, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T18:57:28,971 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d348482, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:57:28,971 DEBUG [Thread-1785 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T18:57:28,971 DEBUG [Thread-1800 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T18:57:28,971 DEBUG [Thread-1794 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T18:57:28,971 DEBUG [Thread-1789 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T18:57:28,972 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T18:57:28,972 DEBUG [Thread-1796 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T18:57:28,972 DEBUG [Thread-1783 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T18:57:28,972 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d79ba0c344fb,43593,1732301724173, seqNum=-1] 2024-11-22T18:57:28,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d79ba0c344fb,43593,1732301724173, seqNum=-1] 2024-11-22T18:57:28,972 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d79ba0c344fb,43593,1732301724173, seqNum=-1] 2024-11-22T18:57:28,972 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d79ba0c344fb,43593,1732301724173, seqNum=-1] 2024-11-22T18:57:28,972 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T18:57:28,972 DEBUG [Thread-1787 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T18:57:28,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T18:57:28,973 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T18:57:28,973 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T18:57:28,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-22T18:57:28,973 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d79ba0c344fb,43593,1732301724173, seqNum=-1] 2024-11-22T18:57:28,973 DEBUG [Thread-1802 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T18:57:28,973 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T18:57:28,974 DEBUG [Thread-1791 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T18:57:28,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d79ba0c344fb,43593,1732301724173, seqNum=-1] 2024-11-22T18:57:28,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-22T18:57:28,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T18:57:28,974 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T18:57:28,974 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d79ba0c344fb,43593,1732301724173, seqNum=-1] 2024-11-22T18:57:28,974 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d79ba0c344fb,43593,1732301724173, seqNum=-1] 2024-11-22T18:57:28,974 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T18:57:28,974 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T18:57:28,975 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d79ba0c344fb,43593,1732301724173, seqNum=-1] 2024-11-22T18:57:28,975 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34978, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T18:57:28,975 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T18:57:28,975 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34994, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T18:57:28,975 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35008, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T18:57:28,975 DEBUG [Thread-1798 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T18:57:28,975 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35032, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T18:57:28,975 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35004, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T18:57:28,976 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35018, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T18:57:28,976 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T18:57:28,976 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35010, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T18:57:28,976 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T18:57:28,976 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35058, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T18:57:28,976 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35042, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T18:57:28,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2] 2024-11-22T18:57:28,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2] 2024-11-22T18:57:28,977 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d79ba0c344fb,43593,1732301724173, seqNum=-1] 2024-11-22T18:57:28,977 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T18:57:28,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2] 2024-11-22T18:57:28,977 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2] 2024-11-22T18:57:28,978 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35074, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T18:57:28,978 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2] 2024-11-22T18:57:28,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2] 2024-11-22T18:57:28,979 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2] 2024-11-22T18:57:28,979 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2] 2024-11-22T18:57:28,980 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2] 2024-11-22T18:57:28,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2] 2024-11-22T18:57:28,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on b35207b85288b14a567277d6fef4534f 2024-11-22T18:57:28,990 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T18:57:28,990 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b35207b85288b14a567277d6fef4534f 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-22T18:57:29,014 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/A/da8513b04689423fbb6d9d225f926ca8 is 50, key is test_row_0/A:col10/1732301848989/Put/seqid=0 2024-11-22T18:57:29,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742218_1394 (size=14341) 2024-11-22T18:57:29,026 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/A/da8513b04689423fbb6d9d225f926ca8 2024-11-22T18:57:29,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:29,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35008 deadline: 1732301909023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:29,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:29,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:29,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1732301909026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:29,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732301909025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:29,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:29,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:29,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:29,037 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:29,037 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:29,037 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:29,037 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:29,037 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:29,037 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:29,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:29,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1732301909027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:29,039 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:29,039 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:29,039 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:29,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:29,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35018 deadline: 1732301909031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:29,040 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:29,041 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:29,041 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:29,047 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/B/c9f7459067154f2390049b737bccf6ed is 50, key is test_row_0/B:col10/1732301848989/Put/seqid=0 2024-11-22T18:57:29,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742219_1395 (size=12001) 2024-11-22T18:57:29,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-22T18:57:29,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-22T18:57:29,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:29,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. as already flushing 2024-11-22T18:57:29,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:29,134 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:29,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:29,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:29,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:29,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35008 deadline: 1732301909145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:29,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:29,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:29,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:29,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:29,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732301909145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:29,153 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:29,153 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:29,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:29,153 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:29,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1732301909146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:29,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:29,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:29,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:29,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:29,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35018 deadline: 1732301909146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:29,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:29,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:29,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:29,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:29,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1732301909146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:29,154 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:29,154 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:29,154 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:29,286 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-22T18:57:29,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:29,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. as already flushing 2024-11-22T18:57:29,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:29,287 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:29,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:29,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:29,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-22T18:57:29,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:29,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35008 deadline: 1732301909355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:29,361 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:29,361 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:29,361 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:29,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:29,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732301909366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:29,372 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:29,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:29,372 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:29,372 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:29,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1732301909366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:29,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:29,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:29,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:29,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:29,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35018 deadline: 1732301909366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:29,372 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:29,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:29,372 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:29,372 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:29,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1732301909366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:29,373 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:29,373 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:29,373 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:29,439 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-22T18:57:29,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:29,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. as already flushing 2024-11-22T18:57:29,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:29,440 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:29,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:29,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:29,453 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/B/c9f7459067154f2390049b737bccf6ed 2024-11-22T18:57:29,472 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/C/624dcfa3c7a44cf495613edcd1359549 is 50, key is test_row_0/C:col10/1732301848989/Put/seqid=0 2024-11-22T18:57:29,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742220_1396 (size=12001) 2024-11-22T18:57:29,591 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-22T18:57:29,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:29,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. as already flushing 2024-11-22T18:57:29,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:29,592 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:29,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:29,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:29,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-22T18:57:29,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:29,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35008 deadline: 1732301909665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:29,673 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:29,673 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:29,674 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:29,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:29,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732301909676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:29,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:29,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:29,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:29,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:29,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1732301909676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:29,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:29,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:29,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:29,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1732301909676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:29,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:29,685 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:29,685 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:29,685 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:29,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:29,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35018 deadline: 1732301909685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:29,688 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:29,689 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:29,689 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:29,744 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-22T18:57:29,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:29,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. as already flushing 2024-11-22T18:57:29,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:29,744 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:29,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:29,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:29,876 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/C/624dcfa3c7a44cf495613edcd1359549 2024-11-22T18:57:29,880 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/A/da8513b04689423fbb6d9d225f926ca8 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/da8513b04689423fbb6d9d225f926ca8 2024-11-22T18:57:29,883 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/da8513b04689423fbb6d9d225f926ca8, entries=200, sequenceid=14, filesize=14.0 K 2024-11-22T18:57:29,884 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/B/c9f7459067154f2390049b737bccf6ed as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/c9f7459067154f2390049b737bccf6ed 2024-11-22T18:57:29,887 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/c9f7459067154f2390049b737bccf6ed, entries=150, sequenceid=14, filesize=11.7 K 2024-11-22T18:57:29,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/C/624dcfa3c7a44cf495613edcd1359549 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/624dcfa3c7a44cf495613edcd1359549 2024-11-22T18:57:29,891 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/624dcfa3c7a44cf495613edcd1359549, entries=150, sequenceid=14, filesize=11.7 K 2024-11-22T18:57:29,892 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for b35207b85288b14a567277d6fef4534f in 902ms, sequenceid=14, compaction requested=false 2024-11-22T18:57:29,892 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-22T18:57:29,893 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b35207b85288b14a567277d6fef4534f: 2024-11-22T18:57:29,897 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-22T18:57:29,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:29,897 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing b35207b85288b14a567277d6fef4534f 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-22T18:57:29,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/A/efbbc5eb3ee446538a289b7f1426a8f8 is 50, key is test_row_0/A:col10/1732301849024/Put/seqid=0 2024-11-22T18:57:29,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742221_1397 (size=12001) 2024-11-22T18:57:29,909 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/A/efbbc5eb3ee446538a289b7f1426a8f8 2024-11-22T18:57:29,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/B/ae8ea60c172e41ef9432695dfbfe4aa6 is 50, key is test_row_0/B:col10/1732301849024/Put/seqid=0 2024-11-22T18:57:29,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742222_1398 (size=12001) 2024-11-22T18:57:30,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-22T18:57:30,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on b35207b85288b14a567277d6fef4534f 2024-11-22T18:57:30,191 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. as already flushing 2024-11-22T18:57:30,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:30,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1732301910203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:30,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:30,214 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:30,214 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:30,214 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:30,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35018 deadline: 1732301910204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:30,214 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:30,214 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:30,214 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:30,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:30,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35008 deadline: 1732301910205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:30,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:30,216 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:30,216 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:30,216 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:30,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732301910207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:30,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:30,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:30,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:30,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:30,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1732301910214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:30,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:30,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:30,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:30,321 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/B/ae8ea60c172e41ef9432695dfbfe4aa6 2024-11-22T18:57:30,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/C/d0df4b25458b41f1abf8bb1d7c1c7667 is 50, key is test_row_0/C:col10/1732301849024/Put/seqid=0 2024-11-22T18:57:30,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:30,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1732301910325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:30,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:30,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35018 deadline: 1732301910325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:30,330 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:30,330 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:30,330 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:30,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:30,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:30,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:30,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:30,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35008 deadline: 1732301910326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:30,331 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:30,331 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:30,331 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:30,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:30,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732301910326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:30,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:30,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:30,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:30,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742223_1399 (size=12001) 2024-11-22T18:57:30,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:30,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1732301910326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:30,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:30,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:30,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:30,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:30,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1732301910535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:30,543 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:30,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:30,543 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:30,543 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:30,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35018 deadline: 1732301910535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:30,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:30,543 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:30,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35008 deadline: 1732301910535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:30,543 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:30,543 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:30,544 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:30,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:30,544 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:30,544 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:30,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732301910536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:30,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:30,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:30,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:30,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:30,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1732301910546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:30,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:30,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:30,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:30,733 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/C/d0df4b25458b41f1abf8bb1d7c1c7667 2024-11-22T18:57:30,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/A/efbbc5eb3ee446538a289b7f1426a8f8 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/efbbc5eb3ee446538a289b7f1426a8f8 2024-11-22T18:57:30,741 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/efbbc5eb3ee446538a289b7f1426a8f8, entries=150, sequenceid=37, filesize=11.7 K 2024-11-22T18:57:30,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/B/ae8ea60c172e41ef9432695dfbfe4aa6 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/ae8ea60c172e41ef9432695dfbfe4aa6 2024-11-22T18:57:30,746 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/ae8ea60c172e41ef9432695dfbfe4aa6, entries=150, sequenceid=37, filesize=11.7 K 2024-11-22T18:57:30,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/C/d0df4b25458b41f1abf8bb1d7c1c7667 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/d0df4b25458b41f1abf8bb1d7c1c7667 2024-11-22T18:57:30,750 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/d0df4b25458b41f1abf8bb1d7c1c7667, entries=150, sequenceid=37, filesize=11.7 K 2024-11-22T18:57:30,751 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for b35207b85288b14a567277d6fef4534f in 854ms, sequenceid=37, compaction requested=false 2024-11-22T18:57:30,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for b35207b85288b14a567277d6fef4534f: 2024-11-22T18:57:30,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:30,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-22T18:57:30,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-22T18:57:30,755 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-22T18:57:30,755 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7770 sec 2024-11-22T18:57:30,756 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.7830 sec 2024-11-22T18:57:30,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on b35207b85288b14a567277d6fef4534f 2024-11-22T18:57:30,852 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T18:57:30,852 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b35207b85288b14a567277d6fef4534f 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-22T18:57:30,858 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/A/9765b061d9ac4549a6e7fc178f7e75e5 is 50, key is test_row_0/A:col10/1732301850852/Put/seqid=0 2024-11-22T18:57:30,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742224_1400 (size=14341) 2024-11-22T18:57:30,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:30,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35018 deadline: 1732301910885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:30,895 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:30,895 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:30,895 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:30,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:30,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35008 deadline: 1732301910891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:30,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:30,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:30,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:30,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1732301910892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:30,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:30,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:30,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:30,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:30,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:30,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1732301910893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:30,906 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:30,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:30,906 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:30,906 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:30,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732301910894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:30,907 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:30,907 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:30,907 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:30,931 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-22T18:57:31,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:31,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35018 deadline: 1732301911006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:31,007 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:31,007 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:31,007 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:31,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:31,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35008 deadline: 1732301911016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:31,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:31,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:31,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:31,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1732301911016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:31,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:31,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:31,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:31,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1732301911016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:31,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:31,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:31,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:31,020 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:31,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732301911016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:31,020 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:31,020 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:31,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:31,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:31,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:31,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-22T18:57:31,125 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T18:57:31,126 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T18:57:31,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-22T18:57:31,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-22T18:57:31,128 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T18:57:31,128 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T18:57:31,129 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T18:57:31,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:31,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35018 deadline: 1732301911216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:31,224 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:31,224 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:31,224 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:31,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:31,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35008 deadline: 1732301911225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:31,229 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:31,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:31,229 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:31,229 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:31,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1732301911225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:31,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:31,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:31,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:31,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-22T18:57:31,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:31,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732301911225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:31,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:31,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:31,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:31,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:31,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1732301911225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:31,237 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:31,237 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:31,237 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:31,263 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/A/9765b061d9ac4549a6e7fc178f7e75e5 2024-11-22T18:57:31,270 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/B/5f3b80c90edb4a728b40fd4aec96d85d is 50, key is test_row_0/B:col10/1732301850852/Put/seqid=0 2024-11-22T18:57:31,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742225_1401 (size=12001) 2024-11-22T18:57:31,281 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-22T18:57:31,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:31,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. as already flushing 2024-11-22T18:57:31,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:31,281 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:31,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:31,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:31,433 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-22T18:57:31,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:31,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. as already flushing 2024-11-22T18:57:31,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:31,434 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:31,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:31,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:31,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-22T18:57:31,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:31,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35018 deadline: 1732301911536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:31,538 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:31,538 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:31,538 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:31,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:31,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35008 deadline: 1732301911536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:31,548 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:31,548 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:31,548 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:31,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:31,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1732301911536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:31,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:31,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:31,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:31,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:31,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732301911545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:31,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:31,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:31,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:31,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:31,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1732301911545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:31,551 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:31,551 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:31,551 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:31,586 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-22T18:57:31,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:31,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. as already flushing 2024-11-22T18:57:31,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:31,587 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:31,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:31,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:31,674 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/B/5f3b80c90edb4a728b40fd4aec96d85d 2024-11-22T18:57:31,681 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/C/8b5843c7b78841f79eb8a737ba7941d8 is 50, key is test_row_0/C:col10/1732301850852/Put/seqid=0 2024-11-22T18:57:31,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742226_1402 (size=12001) 2024-11-22T18:57:31,740 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-22T18:57:31,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:31,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. as already flushing 2024-11-22T18:57:31,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:31,740 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:31,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:31,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:31,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-22T18:57:31,892 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-22T18:57:31,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:31,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. as already flushing 2024-11-22T18:57:31,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:31,893 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:31,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:31,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:32,046 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-22T18:57:32,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:32,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. as already flushing 2024-11-22T18:57:32,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:32,046 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:32,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:32,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:32,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:32,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35018 deadline: 1732301912045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:32,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:32,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:32,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:32,061 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:32,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35008 deadline: 1732301912055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:32,062 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:32,062 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:32,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:32,062 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:32,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1732301912055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:32,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:32,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:32,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:32,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:32,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732301912055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:32,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:32,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:32,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:32,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:32,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1732301912055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:32,063 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:32,063 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:32,063 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:32,085 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/C/8b5843c7b78841f79eb8a737ba7941d8 2024-11-22T18:57:32,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/A/9765b061d9ac4549a6e7fc178f7e75e5 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/9765b061d9ac4549a6e7fc178f7e75e5 2024-11-22T18:57:32,092 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/9765b061d9ac4549a6e7fc178f7e75e5, entries=200, sequenceid=52, filesize=14.0 K 2024-11-22T18:57:32,093 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/B/5f3b80c90edb4a728b40fd4aec96d85d as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/5f3b80c90edb4a728b40fd4aec96d85d 2024-11-22T18:57:32,096 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/5f3b80c90edb4a728b40fd4aec96d85d, entries=150, sequenceid=52, filesize=11.7 K 2024-11-22T18:57:32,097 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/C/8b5843c7b78841f79eb8a737ba7941d8 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/8b5843c7b78841f79eb8a737ba7941d8 2024-11-22T18:57:32,100 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/8b5843c7b78841f79eb8a737ba7941d8, entries=150, sequenceid=52, filesize=11.7 K 2024-11-22T18:57:32,101 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for b35207b85288b14a567277d6fef4534f in 1249ms, sequenceid=52, compaction requested=true 2024-11-22T18:57:32,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b35207b85288b14a567277d6fef4534f: 2024-11-22T18:57:32,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b35207b85288b14a567277d6fef4534f:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T18:57:32,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:57:32,101 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T18:57:32,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b35207b85288b14a567277d6fef4534f:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T18:57:32,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:57:32,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b35207b85288b14a567277d6fef4534f:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T18:57:32,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:57:32,101 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T18:57:32,103 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T18:57:32,103 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1541): b35207b85288b14a567277d6fef4534f/B is initiating minor compaction (all files) 2024-11-22T18:57:32,103 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b35207b85288b14a567277d6fef4534f/B in TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:32,103 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/c9f7459067154f2390049b737bccf6ed, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/ae8ea60c172e41ef9432695dfbfe4aa6, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/5f3b80c90edb4a728b40fd4aec96d85d] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp, totalSize=35.2 K 2024-11-22T18:57:32,103 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T18:57:32,103 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1541): b35207b85288b14a567277d6fef4534f/A is initiating minor compaction (all files) 2024-11-22T18:57:32,103 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b35207b85288b14a567277d6fef4534f/A in TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:32,103 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/da8513b04689423fbb6d9d225f926ca8, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/efbbc5eb3ee446538a289b7f1426a8f8, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/9765b061d9ac4549a6e7fc178f7e75e5] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp, totalSize=39.7 K 2024-11-22T18:57:32,104 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting c9f7459067154f2390049b737bccf6ed, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732301848978 2024-11-22T18:57:32,104 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting da8513b04689423fbb6d9d225f926ca8, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732301848978 2024-11-22T18:57:32,104 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting ae8ea60c172e41ef9432695dfbfe4aa6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732301849024 2024-11-22T18:57:32,104 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting efbbc5eb3ee446538a289b7f1426a8f8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732301849024 2024-11-22T18:57:32,104 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9765b061d9ac4549a6e7fc178f7e75e5, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732301850205 2024-11-22T18:57:32,104 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f3b80c90edb4a728b40fd4aec96d85d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732301850205 2024-11-22T18:57:32,111 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b35207b85288b14a567277d6fef4534f#A#compaction#339 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:57:32,111 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b35207b85288b14a567277d6fef4534f#B#compaction#340 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:57:32,111 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/A/452664526c4d4f879acab6cdf284cebe is 50, key is test_row_0/A:col10/1732301850852/Put/seqid=0 2024-11-22T18:57:32,112 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/B/ea28d406b46a4687a2c3f2700d35ba7a is 50, key is test_row_0/B:col10/1732301850852/Put/seqid=0 2024-11-22T18:57:32,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742227_1403 (size=12104) 2024-11-22T18:57:32,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742228_1404 (size=12104) 2024-11-22T18:57:32,199 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-22T18:57:32,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:32,199 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing b35207b85288b14a567277d6fef4534f 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-22T18:57:32,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/A/37b010b384a143819c2eaccbd7a9555d is 50, key is test_row_0/A:col10/1732301850873/Put/seqid=0 2024-11-22T18:57:32,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742229_1405 (size=12001) 2024-11-22T18:57:32,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-22T18:57:32,525 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/A/452664526c4d4f879acab6cdf284cebe as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/452664526c4d4f879acab6cdf284cebe 2024-11-22T18:57:32,525 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/B/ea28d406b46a4687a2c3f2700d35ba7a as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/ea28d406b46a4687a2c3f2700d35ba7a 2024-11-22T18:57:32,530 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b35207b85288b14a567277d6fef4534f/B of b35207b85288b14a567277d6fef4534f into ea28d406b46a4687a2c3f2700d35ba7a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:57:32,530 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b35207b85288b14a567277d6fef4534f: 2024-11-22T18:57:32,530 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., storeName=b35207b85288b14a567277d6fef4534f/B, priority=13, startTime=1732301852101; duration=0sec 2024-11-22T18:57:32,530 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:57:32,530 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b35207b85288b14a567277d6fef4534f:B 2024-11-22T18:57:32,530 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T18:57:32,530 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b35207b85288b14a567277d6fef4534f/A of b35207b85288b14a567277d6fef4534f into 452664526c4d4f879acab6cdf284cebe(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:57:32,530 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b35207b85288b14a567277d6fef4534f: 2024-11-22T18:57:32,530 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., storeName=b35207b85288b14a567277d6fef4534f/A, priority=13, startTime=1732301852101; duration=0sec 2024-11-22T18:57:32,530 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:57:32,530 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b35207b85288b14a567277d6fef4534f:A 2024-11-22T18:57:32,531 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T18:57:32,531 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1541): b35207b85288b14a567277d6fef4534f/C is initiating minor compaction (all files) 2024-11-22T18:57:32,531 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b35207b85288b14a567277d6fef4534f/C in TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:32,531 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/624dcfa3c7a44cf495613edcd1359549, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/d0df4b25458b41f1abf8bb1d7c1c7667, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/8b5843c7b78841f79eb8a737ba7941d8] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp, totalSize=35.2 K 2024-11-22T18:57:32,531 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 624dcfa3c7a44cf495613edcd1359549, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732301848978 2024-11-22T18:57:32,532 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting d0df4b25458b41f1abf8bb1d7c1c7667, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732301849024 2024-11-22T18:57:32,532 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b5843c7b78841f79eb8a737ba7941d8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732301850205 2024-11-22T18:57:32,539 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b35207b85288b14a567277d6fef4534f#C#compaction#342 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:57:32,539 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/C/30f11592c3524914bb9dc2664a33cc99 is 50, key is test_row_0/C:col10/1732301850852/Put/seqid=0 2024-11-22T18:57:32,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742230_1406 (size=12104) 2024-11-22T18:57:32,608 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/A/37b010b384a143819c2eaccbd7a9555d 2024-11-22T18:57:32,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/B/c9f0bc3608c541f59524119a35696b7c is 50, key is test_row_0/B:col10/1732301850873/Put/seqid=0 2024-11-22T18:57:32,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742231_1407 (size=12001) 2024-11-22T18:57:32,949 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/C/30f11592c3524914bb9dc2664a33cc99 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/30f11592c3524914bb9dc2664a33cc99 2024-11-22T18:57:32,953 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b35207b85288b14a567277d6fef4534f/C of b35207b85288b14a567277d6fef4534f into 30f11592c3524914bb9dc2664a33cc99(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:57:32,953 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b35207b85288b14a567277d6fef4534f: 2024-11-22T18:57:32,953 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., storeName=b35207b85288b14a567277d6fef4534f/C, priority=13, startTime=1732301852101; duration=0sec 2024-11-22T18:57:32,953 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:57:32,953 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b35207b85288b14a567277d6fef4534f:C 2024-11-22T18:57:33,021 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/B/c9f0bc3608c541f59524119a35696b7c 2024-11-22T18:57:33,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/C/afb2eec8ef8e4da498547a0dd1923a4c is 50, key is test_row_0/C:col10/1732301850873/Put/seqid=0 2024-11-22T18:57:33,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742232_1408 (size=12001) 2024-11-22T18:57:33,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on b35207b85288b14a567277d6fef4534f 2024-11-22T18:57:33,068 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. as already flushing 2024-11-22T18:57:33,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:33,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35018 deadline: 1732301913084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:33,094 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:33,094 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:33,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35008 deadline: 1732301913091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:33,094 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:33,094 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:33,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:33,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:33,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:33,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:33,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1732301913092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:33,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:33,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:33,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:33,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732301913092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:33,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:33,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:33,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:33,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:33,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:33,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1732301913093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:33,105 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:33,105 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:33,105 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:33,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:33,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35018 deadline: 1732301913195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:33,197 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:33,197 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:33,197 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:33,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:33,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35008 deadline: 1732301913205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:33,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:33,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:33,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:33,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:33,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1732301913216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:33,221 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:33,221 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:33,221 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:33,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:33,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732301913216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:33,221 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:33,221 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:33,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:33,221 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:33,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1732301913216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:33,222 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:33,222 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:33,222 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:33,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-22T18:57:33,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:33,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35018 deadline: 1732301913405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:33,414 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:33,414 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:33,414 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:33,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:33,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35008 deadline: 1732301913416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:33,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:33,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:33,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:33,433 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/C/afb2eec8ef8e4da498547a0dd1923a4c 2024-11-22T18:57:33,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:33,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1732301913426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:33,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:33,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:33,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:33,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:33,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732301913426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:33,434 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:33,434 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:33,434 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:33,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:33,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1732301913426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:33,434 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:33,434 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:33,434 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:33,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/A/37b010b384a143819c2eaccbd7a9555d as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/37b010b384a143819c2eaccbd7a9555d 2024-11-22T18:57:33,441 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/37b010b384a143819c2eaccbd7a9555d, entries=150, sequenceid=73, filesize=11.7 K 2024-11-22T18:57:33,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/B/c9f0bc3608c541f59524119a35696b7c as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/c9f0bc3608c541f59524119a35696b7c 2024-11-22T18:57:33,445 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/c9f0bc3608c541f59524119a35696b7c, entries=150, sequenceid=73, filesize=11.7 K 2024-11-22T18:57:33,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/C/afb2eec8ef8e4da498547a0dd1923a4c as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/afb2eec8ef8e4da498547a0dd1923a4c 2024-11-22T18:57:33,449 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/afb2eec8ef8e4da498547a0dd1923a4c, entries=150, sequenceid=73, filesize=11.7 K 2024-11-22T18:57:33,450 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for b35207b85288b14a567277d6fef4534f in 1251ms, sequenceid=73, compaction requested=false 2024-11-22T18:57:33,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for b35207b85288b14a567277d6fef4534f: 2024-11-22T18:57:33,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:33,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-22T18:57:33,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-22T18:57:33,453 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-22T18:57:33,453 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3220 sec 2024-11-22T18:57:33,454 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 2.3270 sec 2024-11-22T18:57:33,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on b35207b85288b14a567277d6fef4534f 2024-11-22T18:57:33,721 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T18:57:33,721 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b35207b85288b14a567277d6fef4534f 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-22T18:57:33,725 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/A/2ec8fb05646a4c20b4f1bbbd5e352133 is 50, key is test_row_0/A:col10/1732301853092/Put/seqid=0 2024-11-22T18:57:33,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742233_1409 (size=14341) 2024-11-22T18:57:33,729 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/A/2ec8fb05646a4c20b4f1bbbd5e352133 2024-11-22T18:57:33,736 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/B/55f877c7359b46b08af8089d763c2a87 is 50, key is test_row_0/B:col10/1732301853092/Put/seqid=0 2024-11-22T18:57:33,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742234_1410 (size=12001) 2024-11-22T18:57:33,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:33,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1732301913765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:33,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:33,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:33,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:33,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:33,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1732301913772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:33,778 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:33,778 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:33,778 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:33,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:33,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732301913773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:33,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:33,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:33,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:33,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:33,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35008 deadline: 1732301913774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:33,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:33,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:33,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:33,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:33,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35018 deadline: 1732301913775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:33,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:33,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:33,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:33,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:33,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1732301913885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:33,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:33,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:33,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1732301913885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:33,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:33,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:33,891 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:33,891 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:33,891 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:33,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:33,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732301913895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:33,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:33,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:33,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:33,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:33,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35008 deadline: 1732301913896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:33,904 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:33,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:33,904 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:33,904 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:33,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35018 deadline: 1732301913896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:33,905 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:33,905 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:33,905 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:34,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:34,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1732301914095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:34,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:34,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:34,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:34,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:34,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1732301914095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:34,101 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:34,101 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:34,101 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:34,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:34,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732301914116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:34,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:34,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:34,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:34,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35008 deadline: 1732301914116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:34,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:34,126 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:34,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:34,126 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:34,126 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:34,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35018 deadline: 1732301914116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:34,126 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:34,126 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:34,126 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:34,150 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/B/55f877c7359b46b08af8089d763c2a87 2024-11-22T18:57:34,156 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/C/d82dd5892aa94e91ba025a81eb6e2bcf is 50, key is test_row_0/C:col10/1732301853092/Put/seqid=0 2024-11-22T18:57:34,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742235_1411 (size=12001) 2024-11-22T18:57:34,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:34,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1732301914406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:34,411 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:34,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:34,411 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:34,411 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:34,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1732301914406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:34,411 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:34,411 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:34,411 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:34,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:34,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35008 deadline: 1732301914435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:34,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:34,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:34,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:34,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:34,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732301914435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:34,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:34,444 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:34,444 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:34,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35018 deadline: 1732301914435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:34,444 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:34,445 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:34,445 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:34,445 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:34,560 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/C/d82dd5892aa94e91ba025a81eb6e2bcf 2024-11-22T18:57:34,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/A/2ec8fb05646a4c20b4f1bbbd5e352133 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/2ec8fb05646a4c20b4f1bbbd5e352133 2024-11-22T18:57:34,567 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/2ec8fb05646a4c20b4f1bbbd5e352133, entries=200, sequenceid=92, filesize=14.0 K 2024-11-22T18:57:34,568 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/B/55f877c7359b46b08af8089d763c2a87 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/55f877c7359b46b08af8089d763c2a87 2024-11-22T18:57:34,571 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/55f877c7359b46b08af8089d763c2a87, entries=150, sequenceid=92, filesize=11.7 K 2024-11-22T18:57:34,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/C/d82dd5892aa94e91ba025a81eb6e2bcf as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/d82dd5892aa94e91ba025a81eb6e2bcf 2024-11-22T18:57:34,575 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/d82dd5892aa94e91ba025a81eb6e2bcf, entries=150, sequenceid=92, filesize=11.7 K 2024-11-22T18:57:34,576 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for b35207b85288b14a567277d6fef4534f in 855ms, sequenceid=92, compaction requested=true 2024-11-22T18:57:34,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b35207b85288b14a567277d6fef4534f: 2024-11-22T18:57:34,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b35207b85288b14a567277d6fef4534f:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T18:57:34,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:57:34,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b35207b85288b14a567277d6fef4534f:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T18:57:34,576 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T18:57:34,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:57:34,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b35207b85288b14a567277d6fef4534f:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T18:57:34,576 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T18:57:34,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:57:34,577 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T18:57:34,577 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T18:57:34,577 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1541): b35207b85288b14a567277d6fef4534f/A is initiating minor compaction (all files) 2024-11-22T18:57:34,577 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1541): b35207b85288b14a567277d6fef4534f/B is initiating minor compaction (all files) 2024-11-22T18:57:34,577 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b35207b85288b14a567277d6fef4534f/A in TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:34,577 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b35207b85288b14a567277d6fef4534f/B in TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:34,577 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/452664526c4d4f879acab6cdf284cebe, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/37b010b384a143819c2eaccbd7a9555d, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/2ec8fb05646a4c20b4f1bbbd5e352133] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp, totalSize=37.5 K 2024-11-22T18:57:34,577 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/ea28d406b46a4687a2c3f2700d35ba7a, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/c9f0bc3608c541f59524119a35696b7c, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/55f877c7359b46b08af8089d763c2a87] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp, totalSize=35.3 K 2024-11-22T18:57:34,578 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting ea28d406b46a4687a2c3f2700d35ba7a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732301850205 2024-11-22T18:57:34,578 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 452664526c4d4f879acab6cdf284cebe, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732301850205 2024-11-22T18:57:34,578 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37b010b384a143819c2eaccbd7a9555d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1732301850873 2024-11-22T18:57:34,578 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting c9f0bc3608c541f59524119a35696b7c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1732301850873 2024-11-22T18:57:34,578 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ec8fb05646a4c20b4f1bbbd5e352133, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732301853090 2024-11-22T18:57:34,578 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 55f877c7359b46b08af8089d763c2a87, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732301853090 2024-11-22T18:57:34,584 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b35207b85288b14a567277d6fef4534f#A#compaction#348 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:57:34,584 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b35207b85288b14a567277d6fef4534f#B#compaction#349 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:57:34,584 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/A/98088ebb1f5247a8b6b4334ff635f3ac is 50, key is test_row_0/A:col10/1732301853092/Put/seqid=0 2024-11-22T18:57:34,585 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/B/9af436e10aa84168ab370ac61ef7d40c is 50, key is test_row_0/B:col10/1732301853092/Put/seqid=0 2024-11-22T18:57:34,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742237_1413 (size=12207) 2024-11-22T18:57:34,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742236_1412 (size=12207) 2024-11-22T18:57:34,603 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/A/98088ebb1f5247a8b6b4334ff635f3ac as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/98088ebb1f5247a8b6b4334ff635f3ac 2024-11-22T18:57:34,608 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b35207b85288b14a567277d6fef4534f/A of b35207b85288b14a567277d6fef4534f into 98088ebb1f5247a8b6b4334ff635f3ac(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:57:34,608 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b35207b85288b14a567277d6fef4534f: 2024-11-22T18:57:34,608 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., storeName=b35207b85288b14a567277d6fef4534f/A, priority=13, startTime=1732301854576; duration=0sec 2024-11-22T18:57:34,608 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:57:34,608 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b35207b85288b14a567277d6fef4534f:A 2024-11-22T18:57:34,608 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T18:57:34,609 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T18:57:34,609 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1541): b35207b85288b14a567277d6fef4534f/C is initiating minor compaction (all files) 2024-11-22T18:57:34,609 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b35207b85288b14a567277d6fef4534f/C in TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:34,609 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/30f11592c3524914bb9dc2664a33cc99, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/afb2eec8ef8e4da498547a0dd1923a4c, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/d82dd5892aa94e91ba025a81eb6e2bcf] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp, totalSize=35.3 K 2024-11-22T18:57:34,609 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30f11592c3524914bb9dc2664a33cc99, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732301850205 2024-11-22T18:57:34,610 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting afb2eec8ef8e4da498547a0dd1923a4c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1732301850873 2024-11-22T18:57:34,610 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting d82dd5892aa94e91ba025a81eb6e2bcf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732301853090 2024-11-22T18:57:34,616 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b35207b85288b14a567277d6fef4534f#C#compaction#350 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:57:34,616 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/C/6dcada98e12f4bdcab4034a2852dcdcc is 50, key is test_row_0/C:col10/1732301853092/Put/seqid=0 2024-11-22T18:57:34,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742238_1414 (size=12207) 2024-11-22T18:57:34,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on b35207b85288b14a567277d6fef4534f 2024-11-22T18:57:34,922 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T18:57:34,922 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b35207b85288b14a567277d6fef4534f 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-22T18:57:34,926 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/A/864d5c4314444664a88f487e6e67ae82 is 50, key is test_row_0/A:col10/1732301853773/Put/seqid=0 2024-11-22T18:57:34,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742239_1415 (size=14341) 2024-11-22T18:57:34,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:34,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35018 deadline: 1732301914956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:34,965 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:34,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:34,965 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:34,965 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:34,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1732301914959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:34,965 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:34,966 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:34,966 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:34,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:34,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1732301914962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:34,974 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:34,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:34,974 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:34,974 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:34,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35008 deadline: 1732301914964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:34,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:34,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:34,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:34,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:34,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732301914965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:34,978 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:34,978 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:34,978 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:34,998 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/B/9af436e10aa84168ab370ac61ef7d40c as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/9af436e10aa84168ab370ac61ef7d40c 2024-11-22T18:57:35,002 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b35207b85288b14a567277d6fef4534f/B of b35207b85288b14a567277d6fef4534f into 9af436e10aa84168ab370ac61ef7d40c(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:57:35,002 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b35207b85288b14a567277d6fef4534f: 2024-11-22T18:57:35,002 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., storeName=b35207b85288b14a567277d6fef4534f/B, priority=13, startTime=1732301854576; duration=0sec 2024-11-22T18:57:35,002 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:57:35,002 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b35207b85288b14a567277d6fef4534f:B 2024-11-22T18:57:35,024 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/C/6dcada98e12f4bdcab4034a2852dcdcc as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/6dcada98e12f4bdcab4034a2852dcdcc 2024-11-22T18:57:35,028 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b35207b85288b14a567277d6fef4534f/C of b35207b85288b14a567277d6fef4534f into 6dcada98e12f4bdcab4034a2852dcdcc(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:57:35,028 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b35207b85288b14a567277d6fef4534f: 2024-11-22T18:57:35,028 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., storeName=b35207b85288b14a567277d6fef4534f/C, priority=13, startTime=1732301854576; duration=0sec 2024-11-22T18:57:35,028 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:57:35,028 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b35207b85288b14a567277d6fef4534f:C 2024-11-22T18:57:35,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:35,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1732301915076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:35,083 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:35,083 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:35,083 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:35,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:35,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1732301915086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:35,095 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:35,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:35,095 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:35,095 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:35,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35008 deadline: 1732301915086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:35,096 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:35,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:35,096 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:35,096 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:35,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732301915086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:35,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:35,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:35,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:35,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-22T18:57:35,286 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T18:57:35,286 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T18:57:35,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-22T18:57:35,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-22T18:57:35,288 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T18:57:35,289 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T18:57:35,289 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T18:57:35,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:35,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1732301915286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:35,292 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:35,292 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:35,292 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:35,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:35,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1732301915305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:35,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:35,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:35,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35008 deadline: 1732301915305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:35,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:35,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:35,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:35,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:35,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:35,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:35,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732301915305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:35,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:35,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:35,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:35,330 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/A/864d5c4314444664a88f487e6e67ae82 2024-11-22T18:57:35,337 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/B/3428fa5b5e3343099e8b7b88e8051792 is 50, key is test_row_0/B:col10/1732301853773/Put/seqid=0 2024-11-22T18:57:35,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742240_1416 (size=12001) 2024-11-22T18:57:35,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-22T18:57:35,441 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-22T18:57:35,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:35,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. as already flushing 2024-11-22T18:57:35,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:35,441 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:35,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:35,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:35,594 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-22T18:57:35,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:35,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. as already flushing 2024-11-22T18:57:35,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:35,594 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:35,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:35,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:35,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:35,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1732301915596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:35,601 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:35,601 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:35,601 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:35,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-22T18:57:35,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:35,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1732301915625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:35,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:35,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:35,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35008 deadline: 1732301915625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:35,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:35,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:35,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:35,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:35,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:35,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:35,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732301915625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:35,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:35,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:35,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:35,741 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/B/3428fa5b5e3343099e8b7b88e8051792 2024-11-22T18:57:35,747 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-22T18:57:35,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:35,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. as already flushing 2024-11-22T18:57:35,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:35,747 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:35,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:35,748 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/C/a7f23032ef694650975ecab2352cd341 is 50, key is test_row_0/C:col10/1732301853773/Put/seqid=0 2024-11-22T18:57:35,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:35,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742241_1417 (size=12001) 2024-11-22T18:57:35,900 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-22T18:57:35,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:35,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. as already flushing 2024-11-22T18:57:35,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:35,900 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:35,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:35,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:35,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-22T18:57:35,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:35,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35018 deadline: 1732301915975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:35,984 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to d79ba0c344fb,43593,1732301724173 for 'test_row_2' in b35207b85288b14a567277d6fef4534f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] ... 32 more 2024-11-22T18:57:35,985 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:35,985 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:35,985 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:36,053 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-22T18:57:36,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:36,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. as already flushing 2024-11-22T18:57:36,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:36,053 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:36,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:36,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:36,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:36,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1732301916115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:36,123 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:36,123 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:36,123 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:36,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:36,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1732301916135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:36,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:36,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732301916135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:36,143 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:36,143 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:36,143 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:36,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:36,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:36,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:36,152 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/C/a7f23032ef694650975ecab2352cd341 2024-11-22T18:57:36,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:36,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35008 deadline: 1732301916146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:36,157 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/A/864d5c4314444664a88f487e6e67ae82 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/864d5c4314444664a88f487e6e67ae82 2024-11-22T18:57:36,157 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:36,157 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:36,157 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:36,161 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/864d5c4314444664a88f487e6e67ae82, entries=200, sequenceid=114, filesize=14.0 K 2024-11-22T18:57:36,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/B/3428fa5b5e3343099e8b7b88e8051792 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/3428fa5b5e3343099e8b7b88e8051792 2024-11-22T18:57:36,168 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/3428fa5b5e3343099e8b7b88e8051792, entries=150, sequenceid=114, filesize=11.7 K 2024-11-22T18:57:36,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/C/a7f23032ef694650975ecab2352cd341 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/a7f23032ef694650975ecab2352cd341 2024-11-22T18:57:36,175 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/a7f23032ef694650975ecab2352cd341, entries=150, sequenceid=114, filesize=11.7 K 2024-11-22T18:57:36,176 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for b35207b85288b14a567277d6fef4534f in 1254ms, sequenceid=114, compaction requested=false 2024-11-22T18:57:36,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b35207b85288b14a567277d6fef4534f: 2024-11-22T18:57:36,206 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-22T18:57:36,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:36,206 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing b35207b85288b14a567277d6fef4534f 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-22T18:57:36,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/A/e9a06511de1c446ca8ebd9d0b1590344 is 50, key is test_row_0/A:col10/1732301854963/Put/seqid=0 2024-11-22T18:57:36,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742242_1418 (size=12001) 2024-11-22T18:57:36,222 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/A/e9a06511de1c446ca8ebd9d0b1590344 2024-11-22T18:57:36,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/B/a16133ab1d7a48928d55bbaeb5fcd030 is 50, key is test_row_0/B:col10/1732301854963/Put/seqid=0 2024-11-22T18:57:36,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742243_1419 (size=12001) 2024-11-22T18:57:36,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-22T18:57:36,635 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/B/a16133ab1d7a48928d55bbaeb5fcd030 2024-11-22T18:57:36,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/C/2506525daf5844c498b51392b7f15bb4 is 50, key is test_row_0/C:col10/1732301854963/Put/seqid=0 2024-11-22T18:57:36,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742244_1420 (size=12001) 2024-11-22T18:57:36,651 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/C/2506525daf5844c498b51392b7f15bb4 2024-11-22T18:57:36,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/A/e9a06511de1c446ca8ebd9d0b1590344 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/e9a06511de1c446ca8ebd9d0b1590344 2024-11-22T18:57:36,660 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/e9a06511de1c446ca8ebd9d0b1590344, entries=150, sequenceid=131, filesize=11.7 K 2024-11-22T18:57:36,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/B/a16133ab1d7a48928d55bbaeb5fcd030 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/a16133ab1d7a48928d55bbaeb5fcd030 2024-11-22T18:57:36,664 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/a16133ab1d7a48928d55bbaeb5fcd030, entries=150, sequenceid=131, filesize=11.7 K 2024-11-22T18:57:36,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/C/2506525daf5844c498b51392b7f15bb4 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/2506525daf5844c498b51392b7f15bb4 2024-11-22T18:57:36,668 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/2506525daf5844c498b51392b7f15bb4, entries=150, sequenceid=131, filesize=11.7 K 2024-11-22T18:57:36,669 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=0 B/0 for b35207b85288b14a567277d6fef4534f in 463ms, sequenceid=131, compaction requested=true 2024-11-22T18:57:36,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for b35207b85288b14a567277d6fef4534f: 2024-11-22T18:57:36,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:36,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-22T18:57:36,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-22T18:57:36,672 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-22T18:57:36,673 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3820 sec 2024-11-22T18:57:36,674 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 1.3860 sec 2024-11-22T18:57:37,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on b35207b85288b14a567277d6fef4534f 2024-11-22T18:57:37,160 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T18:57:37,160 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b35207b85288b14a567277d6fef4534f 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-22T18:57:37,165 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/A/0166be198db14acb88971e0eb795b426 is 50, key is test_row_0/A:col10/1732301857156/Put/seqid=0 2024-11-22T18:57:37,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742245_1421 (size=16931) 2024-11-22T18:57:37,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:37,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1732301917216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:37,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:37,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1732301917220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:37,230 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:37,230 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:37,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:37,230 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:37,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732301917220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:37,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:37,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:37,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:37,230 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:37,230 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:37,230 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:37,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:37,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35008 deadline: 1732301917230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:37,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:37,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:37,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:37,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:37,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1732301917335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:37,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:37,346 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:37,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1732301917335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:37,346 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:37,346 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:37,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:37,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:37,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:37,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:37,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35008 deadline: 1732301917335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:37,346 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:37,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:37,346 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:37,346 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:37,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732301917335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:37,347 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:37,347 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:37,347 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:37,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-22T18:57:37,446 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T18:57:37,446 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T18:57:37,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-11-22T18:57:37,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-22T18:57:37,448 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T18:57:37,449 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T18:57:37,449 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T18:57:37,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-22T18:57:37,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:37,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1732301917555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:37,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:37,557 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:37,557 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:37,557 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:37,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1732301917555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:37,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:37,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:37,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:37,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35008 deadline: 1732301917556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:37,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:37,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:37,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:37,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:37,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:37,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732301917556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:37,566 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:37,566 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:37,566 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:37,583 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/A/0166be198db14acb88971e0eb795b426 2024-11-22T18:57:37,590 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/B/cd73d11138e746efabd424667d56bdaf is 50, key is test_row_0/B:col10/1732301857156/Put/seqid=0 2024-11-22T18:57:37,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742246_1422 (size=12151) 2024-11-22T18:57:37,596 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/B/cd73d11138e746efabd424667d56bdaf 2024-11-22T18:57:37,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-22T18:57:37,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:37,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. as already flushing 2024-11-22T18:57:37,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:37,601 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:37,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:37,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:37,604 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/C/76d82c8937ce42ebb82235d1e1c62ac4 is 50, key is test_row_0/C:col10/1732301857156/Put/seqid=0 2024-11-22T18:57:37,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742247_1423 (size=12151) 2024-11-22T18:57:37,753 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-22T18:57:37,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:37,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. as already flushing 2024-11-22T18:57:37,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:37,754 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:37,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:37,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:37,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-22T18:57:37,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:37,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1732301917865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:37,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:37,871 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:37,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1732301917865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:37,871 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:37,871 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:37,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:37,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:37,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:37,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35008 deadline: 1732301917865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:37,871 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:37,872 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:37,872 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:37,872 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:37,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:37,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732301917875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:37,885 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:37,885 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:37,885 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:37,906 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-22T18:57:37,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:37,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. as already flushing 2024-11-22T18:57:37,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:37,907 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:37,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:37,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:38,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:38,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35018 deadline: 1732301917995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:38,001 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to d79ba0c344fb,43593,1732301724173 for 'test_row_2' in b35207b85288b14a567277d6fef4534f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4226 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] ... 32 more 2024-11-22T18:57:38,001 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:38,001 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:38,001 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:38,009 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/C/76d82c8937ce42ebb82235d1e1c62ac4 2024-11-22T18:57:38,013 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/A/0166be198db14acb88971e0eb795b426 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/0166be198db14acb88971e0eb795b426 2024-11-22T18:57:38,017 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/0166be198db14acb88971e0eb795b426, entries=250, sequenceid=142, filesize=16.5 K 2024-11-22T18:57:38,018 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/B/cd73d11138e746efabd424667d56bdaf as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/cd73d11138e746efabd424667d56bdaf 2024-11-22T18:57:38,022 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/cd73d11138e746efabd424667d56bdaf, entries=150, sequenceid=142, filesize=11.9 K 2024-11-22T18:57:38,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/C/76d82c8937ce42ebb82235d1e1c62ac4 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/76d82c8937ce42ebb82235d1e1c62ac4 2024-11-22T18:57:38,027 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/76d82c8937ce42ebb82235d1e1c62ac4, entries=150, sequenceid=142, filesize=11.9 K 2024-11-22T18:57:38,028 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for b35207b85288b14a567277d6fef4534f in 868ms, sequenceid=142, compaction requested=true 2024-11-22T18:57:38,029 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b35207b85288b14a567277d6fef4534f: 2024-11-22T18:57:38,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b35207b85288b14a567277d6fef4534f:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T18:57:38,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:57:38,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b35207b85288b14a567277d6fef4534f:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T18:57:38,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:57:38,029 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T18:57:38,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b35207b85288b14a567277d6fef4534f:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T18:57:38,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:57:38,029 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T18:57:38,030 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T18:57:38,030 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1541): b35207b85288b14a567277d6fef4534f/B is initiating minor compaction (all files) 2024-11-22T18:57:38,030 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b35207b85288b14a567277d6fef4534f/B in TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:38,030 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/9af436e10aa84168ab370ac61ef7d40c, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/3428fa5b5e3343099e8b7b88e8051792, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/a16133ab1d7a48928d55bbaeb5fcd030, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/cd73d11138e746efabd424667d56bdaf] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp, totalSize=47.2 K 2024-11-22T18:57:38,030 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55480 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T18:57:38,031 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1541): b35207b85288b14a567277d6fef4534f/A is initiating minor compaction (all files) 2024-11-22T18:57:38,031 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b35207b85288b14a567277d6fef4534f/A in TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:38,031 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/98088ebb1f5247a8b6b4334ff635f3ac, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/864d5c4314444664a88f487e6e67ae82, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/e9a06511de1c446ca8ebd9d0b1590344, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/0166be198db14acb88971e0eb795b426] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp, totalSize=54.2 K 2024-11-22T18:57:38,031 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 9af436e10aa84168ab370ac61ef7d40c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732301853090 2024-11-22T18:57:38,031 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98088ebb1f5247a8b6b4334ff635f3ac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732301853090 2024-11-22T18:57:38,031 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 3428fa5b5e3343099e8b7b88e8051792, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732301853772 2024-11-22T18:57:38,032 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 864d5c4314444664a88f487e6e67ae82, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732301853771 2024-11-22T18:57:38,032 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting a16133ab1d7a48928d55bbaeb5fcd030, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732301854961 2024-11-22T18:57:38,032 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9a06511de1c446ca8ebd9d0b1590344, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732301854961 2024-11-22T18:57:38,032 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting cd73d11138e746efabd424667d56bdaf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1732301857144 2024-11-22T18:57:38,032 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0166be198db14acb88971e0eb795b426, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1732301857141 2024-11-22T18:57:38,042 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b35207b85288b14a567277d6fef4534f#A#compaction#360 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:57:38,042 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/A/177ff348c93d47c0af3c72b43ce3e0e9 is 50, key is test_row_0/A:col10/1732301857156/Put/seqid=0 2024-11-22T18:57:38,042 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b35207b85288b14a567277d6fef4534f#B#compaction#361 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:57:38,043 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/B/325de31a960245e8a960585f2d81f752 is 50, key is test_row_0/B:col10/1732301857156/Put/seqid=0 2024-11-22T18:57:38,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742249_1425 (size=12493) 2024-11-22T18:57:38,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742248_1424 (size=12493) 2024-11-22T18:57:38,059 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/B/325de31a960245e8a960585f2d81f752 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/325de31a960245e8a960585f2d81f752 2024-11-22T18:57:38,060 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-22T18:57:38,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:38,060 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing b35207b85288b14a567277d6fef4534f 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-22T18:57:38,063 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b35207b85288b14a567277d6fef4534f/B of b35207b85288b14a567277d6fef4534f into 325de31a960245e8a960585f2d81f752(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:57:38,063 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b35207b85288b14a567277d6fef4534f: 2024-11-22T18:57:38,063 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., storeName=b35207b85288b14a567277d6fef4534f/B, priority=12, startTime=1732301858029; duration=0sec 2024-11-22T18:57:38,063 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:57:38,064 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b35207b85288b14a567277d6fef4534f:B 2024-11-22T18:57:38,064 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T18:57:38,065 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T18:57:38,065 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1541): b35207b85288b14a567277d6fef4534f/C is initiating minor compaction (all files) 2024-11-22T18:57:38,065 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b35207b85288b14a567277d6fef4534f/C in TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:38,065 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/6dcada98e12f4bdcab4034a2852dcdcc, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/a7f23032ef694650975ecab2352cd341, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/2506525daf5844c498b51392b7f15bb4, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/76d82c8937ce42ebb82235d1e1c62ac4] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp, totalSize=47.2 K 2024-11-22T18:57:38,065 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 6dcada98e12f4bdcab4034a2852dcdcc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732301853090 2024-11-22T18:57:38,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/A/36d9b8ae99c945c5b206130cf792fe6f is 50, key is test_row_0/A:col10/1732301857196/Put/seqid=0 2024-11-22T18:57:38,065 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting a7f23032ef694650975ecab2352cd341, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732301853772 2024-11-22T18:57:38,066 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 2506525daf5844c498b51392b7f15bb4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732301854961 2024-11-22T18:57:38,066 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 76d82c8937ce42ebb82235d1e1c62ac4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1732301857144 2024-11-22T18:57:38,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742250_1426 (size=12151) 2024-11-22T18:57:38,070 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/A/36d9b8ae99c945c5b206130cf792fe6f 2024-11-22T18:57:38,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-22T18:57:38,076 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b35207b85288b14a567277d6fef4534f#C#compaction#363 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:57:38,077 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/C/cf6c3a6fd0214c53aa6d40627b04cd6b is 50, key is test_row_0/C:col10/1732301857156/Put/seqid=0 2024-11-22T18:57:38,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/B/5362013ac71f4d5cad07586e3e6ae640 is 50, key is test_row_0/B:col10/1732301857196/Put/seqid=0 2024-11-22T18:57:38,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742252_1428 (size=12151) 2024-11-22T18:57:38,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742251_1427 (size=12493) 2024-11-22T18:57:38,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on b35207b85288b14a567277d6fef4534f 2024-11-22T18:57:38,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. as already flushing 2024-11-22T18:57:38,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:38,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732301918396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:38,405 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:38,405 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:38,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:38,405 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:38,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1732301918398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:38,406 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:38,406 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:38,406 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:38,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:38,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1732301918404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:38,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:38,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:38,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:38,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:38,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35008 deadline: 1732301918405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:38,416 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:38,416 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:38,416 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:38,460 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/A/177ff348c93d47c0af3c72b43ce3e0e9 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/177ff348c93d47c0af3c72b43ce3e0e9 2024-11-22T18:57:38,463 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b35207b85288b14a567277d6fef4534f/A of b35207b85288b14a567277d6fef4534f into 177ff348c93d47c0af3c72b43ce3e0e9(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:57:38,463 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b35207b85288b14a567277d6fef4534f: 2024-11-22T18:57:38,463 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., storeName=b35207b85288b14a567277d6fef4534f/A, priority=12, startTime=1732301858029; duration=0sec 2024-11-22T18:57:38,464 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:57:38,464 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b35207b85288b14a567277d6fef4534f:A 2024-11-22T18:57:38,486 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/B/5362013ac71f4d5cad07586e3e6ae640 2024-11-22T18:57:38,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/C/74c0e30478f64fe3a5b0bac7d175fca1 is 50, key is test_row_0/C:col10/1732301857196/Put/seqid=0 2024-11-22T18:57:38,494 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/C/cf6c3a6fd0214c53aa6d40627b04cd6b as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/cf6c3a6fd0214c53aa6d40627b04cd6b 2024-11-22T18:57:38,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742253_1429 (size=12151) 2024-11-22T18:57:38,500 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b35207b85288b14a567277d6fef4534f/C of b35207b85288b14a567277d6fef4534f into cf6c3a6fd0214c53aa6d40627b04cd6b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:57:38,500 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b35207b85288b14a567277d6fef4534f: 2024-11-22T18:57:38,500 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., storeName=b35207b85288b14a567277d6fef4534f/C, priority=12, startTime=1732301858029; duration=0sec 2024-11-22T18:57:38,500 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:57:38,500 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b35207b85288b14a567277d6fef4534f:C 2024-11-22T18:57:38,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:38,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1732301918516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:38,524 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:38,524 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:38,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1732301918516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:38,524 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:38,524 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:38,524 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:38,524 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:38,524 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:38,534 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:38,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35008 deadline: 1732301918526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:38,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:38,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:38,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:38,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-22T18:57:38,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:38,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1732301918736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:38,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:38,741 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:38,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1732301918736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:38,741 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:38,741 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:38,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:38,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:38,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:38,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:38,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35008 deadline: 1732301918746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:38,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:38,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:38,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:38,897 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/C/74c0e30478f64fe3a5b0bac7d175fca1 2024-11-22T18:57:38,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/A/36d9b8ae99c945c5b206130cf792fe6f as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/36d9b8ae99c945c5b206130cf792fe6f 2024-11-22T18:57:38,905 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/36d9b8ae99c945c5b206130cf792fe6f, entries=150, sequenceid=167, filesize=11.9 K 2024-11-22T18:57:38,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/B/5362013ac71f4d5cad07586e3e6ae640 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/5362013ac71f4d5cad07586e3e6ae640 2024-11-22T18:57:38,908 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/5362013ac71f4d5cad07586e3e6ae640, entries=150, sequenceid=167, filesize=11.9 K 2024-11-22T18:57:38,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/C/74c0e30478f64fe3a5b0bac7d175fca1 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/74c0e30478f64fe3a5b0bac7d175fca1 2024-11-22T18:57:38,912 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/74c0e30478f64fe3a5b0bac7d175fca1, entries=150, sequenceid=167, filesize=11.9 K 2024-11-22T18:57:38,913 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for b35207b85288b14a567277d6fef4534f in 853ms, sequenceid=167, compaction requested=false 2024-11-22T18:57:38,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for b35207b85288b14a567277d6fef4534f: 2024-11-22T18:57:38,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:38,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-11-22T18:57:38,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-11-22T18:57:38,917 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-22T18:57:38,917 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4660 sec 2024-11-22T18:57:38,918 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 1.4710 sec 2024-11-22T18:57:39,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on b35207b85288b14a567277d6fef4534f 2024-11-22T18:57:39,051 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T18:57:39,051 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b35207b85288b14a567277d6fef4534f 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-22T18:57:39,055 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/A/6d7d921517c94df7a5070518c4e63110 is 50, key is test_row_0/A:col10/1732301859049/Put/seqid=0 2024-11-22T18:57:39,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742254_1430 (size=14541) 2024-11-22T18:57:39,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:39,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1732301919109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:39,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:39,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:39,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:39,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:39,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35008 deadline: 1732301919110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:39,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:39,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1732301919113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:39,122 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:39,122 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:39,122 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:39,122 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:39,123 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:39,123 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:39,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:39,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1732301919226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:39,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:39,234 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:39,234 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:39,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35008 deadline: 1732301919226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:39,234 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:39,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:39,234 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:39,234 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:39,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1732301919226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:39,235 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:39,235 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:39,235 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:39,235 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:39,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:39,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732301919415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:39,422 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to d79ba0c344fb,43593,1732301724173 for 'test_row_0' in b35207b85288b14a567277d6fef4534f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] ... 32 more 2024-11-22T18:57:39,422 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:39,422 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:39,422 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:39,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:39,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1732301919446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:39,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:39,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:39,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:39,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:39,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35008 deadline: 1732301919446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:39,457 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:39,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:39,457 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:39,457 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:39,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1732301919446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:39,457 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:39,458 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:39,458 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:39,463 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/A/6d7d921517c94df7a5070518c4e63110 2024-11-22T18:57:39,470 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/B/11deadd861404b208dbdde8e4b4aa851 is 50, key is test_row_0/B:col10/1732301859049/Put/seqid=0 2024-11-22T18:57:39,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742255_1431 (size=12151) 2024-11-22T18:57:39,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-22T18:57:39,595 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T18:57:39,596 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T18:57:39,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees 2024-11-22T18:57:39,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-22T18:57:39,597 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T18:57:39,598 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T18:57:39,598 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T18:57:39,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-22T18:57:39,750 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-22T18:57:39,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:39,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. as already flushing 2024-11-22T18:57:39,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:39,750 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:39,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:39,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:39,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:39,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1732301919765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:39,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:39,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:39,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:39,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:39,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35008 deadline: 1732301919765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:39,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:39,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:39,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:39,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:39,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1732301919765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:39,773 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:39,773 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:39,773 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:39,875 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/B/11deadd861404b208dbdde8e4b4aa851 2024-11-22T18:57:39,883 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/C/18aa90f3e39947b58cc653b3acbae2df is 50, key is test_row_0/C:col10/1732301859049/Put/seqid=0 2024-11-22T18:57:39,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742256_1432 (size=12151) 2024-11-22T18:57:39,903 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-22T18:57:39,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:39,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. as already flushing 2024-11-22T18:57:39,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:39,903 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:39,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:39,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:39,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-22T18:57:40,056 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-22T18:57:40,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:40,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. as already flushing 2024-11-22T18:57:40,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:40,057 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:40,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:40,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:40,209 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-22T18:57:40,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:40,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. as already flushing 2024-11-22T18:57:40,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:40,210 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:40,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:40,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:40,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-22T18:57:40,287 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/C/18aa90f3e39947b58cc653b3acbae2df 2024-11-22T18:57:40,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/A/6d7d921517c94df7a5070518c4e63110 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/6d7d921517c94df7a5070518c4e63110 2024-11-22T18:57:40,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:40,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1732301920286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:40,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:40,294 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:40,294 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:40,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35008 deadline: 1732301920286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:40,294 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:40,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:40,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:40,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:40,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1732301920286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:40,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:40,294 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:40,294 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:40,294 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:40,295 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/6d7d921517c94df7a5070518c4e63110, entries=200, sequenceid=182, filesize=14.2 K 2024-11-22T18:57:40,295 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/B/11deadd861404b208dbdde8e4b4aa851 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/11deadd861404b208dbdde8e4b4aa851 2024-11-22T18:57:40,299 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/11deadd861404b208dbdde8e4b4aa851, entries=150, sequenceid=182, filesize=11.9 K 2024-11-22T18:57:40,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/C/18aa90f3e39947b58cc653b3acbae2df as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/18aa90f3e39947b58cc653b3acbae2df 2024-11-22T18:57:40,302 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/18aa90f3e39947b58cc653b3acbae2df, entries=150, sequenceid=182, filesize=11.9 K 2024-11-22T18:57:40,303 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for b35207b85288b14a567277d6fef4534f in 1252ms, sequenceid=182, compaction requested=true 2024-11-22T18:57:40,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b35207b85288b14a567277d6fef4534f: 2024-11-22T18:57:40,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b35207b85288b14a567277d6fef4534f:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T18:57:40,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:57:40,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b35207b85288b14a567277d6fef4534f:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T18:57:40,303 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T18:57:40,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:57:40,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b35207b85288b14a567277d6fef4534f:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T18:57:40,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:57:40,303 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T18:57:40,304 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T18:57:40,304 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T18:57:40,304 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1541): b35207b85288b14a567277d6fef4534f/A is initiating minor compaction (all files) 2024-11-22T18:57:40,304 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1541): b35207b85288b14a567277d6fef4534f/B is initiating minor compaction (all files) 2024-11-22T18:57:40,304 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b35207b85288b14a567277d6fef4534f/A in TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:40,304 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b35207b85288b14a567277d6fef4534f/B in TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:40,304 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/177ff348c93d47c0af3c72b43ce3e0e9, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/36d9b8ae99c945c5b206130cf792fe6f, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/6d7d921517c94df7a5070518c4e63110] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp, totalSize=38.3 K 2024-11-22T18:57:40,304 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/325de31a960245e8a960585f2d81f752, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/5362013ac71f4d5cad07586e3e6ae640, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/11deadd861404b208dbdde8e4b4aa851] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp, totalSize=35.9 K 2024-11-22T18:57:40,305 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 177ff348c93d47c0af3c72b43ce3e0e9, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1732301857144 2024-11-22T18:57:40,305 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 325de31a960245e8a960585f2d81f752, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1732301857144 2024-11-22T18:57:40,305 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36d9b8ae99c945c5b206130cf792fe6f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732301857196 2024-11-22T18:57:40,305 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 5362013ac71f4d5cad07586e3e6ae640, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732301857196 2024-11-22T18:57:40,305 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d7d921517c94df7a5070518c4e63110, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1732301858393 2024-11-22T18:57:40,305 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 11deadd861404b208dbdde8e4b4aa851, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1732301858393 2024-11-22T18:57:40,320 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b35207b85288b14a567277d6fef4534f#B#compaction#369 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:57:40,320 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b35207b85288b14a567277d6fef4534f#A#compaction#370 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:57:40,321 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/B/e39c40b8e4684fbca85fb263ec7d5f96 is 50, key is test_row_0/B:col10/1732301859049/Put/seqid=0 2024-11-22T18:57:40,321 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/A/495816b50c274d54b540ded95179125e is 50, key is test_row_0/A:col10/1732301859049/Put/seqid=0 2024-11-22T18:57:40,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742257_1433 (size=12595) 2024-11-22T18:57:40,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742258_1434 (size=12595) 2024-11-22T18:57:40,362 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-22T18:57:40,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:40,363 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2902): Flushing b35207b85288b14a567277d6fef4534f 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-22T18:57:40,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/A/6ff02883b40549b5a870ad8ddaed5b16 is 50, key is test_row_0/A:col10/1732301859109/Put/seqid=0 2024-11-22T18:57:40,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742259_1435 (size=12151) 2024-11-22T18:57:40,377 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/A/6ff02883b40549b5a870ad8ddaed5b16 2024-11-22T18:57:40,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/B/924563c13d0549c4a6015887cac32305 is 50, key is test_row_0/B:col10/1732301859109/Put/seqid=0 2024-11-22T18:57:40,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742260_1436 (size=12151) 2024-11-22T18:57:40,394 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/B/924563c13d0549c4a6015887cac32305 2024-11-22T18:57:40,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/C/418db2535f394a099c0b45e149f2f446 is 50, key is test_row_0/C:col10/1732301859109/Put/seqid=0 2024-11-22T18:57:40,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742261_1437 (size=12151) 2024-11-22T18:57:40,407 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/C/418db2535f394a099c0b45e149f2f446 2024-11-22T18:57:40,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/A/6ff02883b40549b5a870ad8ddaed5b16 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/6ff02883b40549b5a870ad8ddaed5b16 2024-11-22T18:57:40,417 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/6ff02883b40549b5a870ad8ddaed5b16, entries=150, sequenceid=206, filesize=11.9 K 2024-11-22T18:57:40,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/B/924563c13d0549c4a6015887cac32305 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/924563c13d0549c4a6015887cac32305 2024-11-22T18:57:40,423 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/924563c13d0549c4a6015887cac32305, entries=150, sequenceid=206, filesize=11.9 K 2024-11-22T18:57:40,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/C/418db2535f394a099c0b45e149f2f446 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/418db2535f394a099c0b45e149f2f446 2024-11-22T18:57:40,428 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/418db2535f394a099c0b45e149f2f446, entries=150, sequenceid=206, filesize=11.9 K 2024-11-22T18:57:40,429 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=0 B/0 for b35207b85288b14a567277d6fef4534f in 67ms, sequenceid=206, compaction requested=true 2024-11-22T18:57:40,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2603): Flush status journal for b35207b85288b14a567277d6fef4534f: 2024-11-22T18:57:40,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:40,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=140 2024-11-22T18:57:40,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4169): Remote procedure done, pid=140 2024-11-22T18:57:40,435 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-22T18:57:40,435 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 833 msec 2024-11-22T18:57:40,436 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees in 839 msec 2024-11-22T18:57:40,735 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/B/e39c40b8e4684fbca85fb263ec7d5f96 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/e39c40b8e4684fbca85fb263ec7d5f96 2024-11-22T18:57:40,735 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/A/495816b50c274d54b540ded95179125e as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/495816b50c274d54b540ded95179125e 2024-11-22T18:57:40,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-22T18:57:40,735 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T18:57:40,736 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T18:57:40,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees 2024-11-22T18:57:40,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-22T18:57:40,738 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=141, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T18:57:40,739 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=141, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T18:57:40,739 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T18:57:40,740 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b35207b85288b14a567277d6fef4534f/B of b35207b85288b14a567277d6fef4534f into e39c40b8e4684fbca85fb263ec7d5f96(size=12.3 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:57:40,740 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b35207b85288b14a567277d6fef4534f: 2024-11-22T18:57:40,740 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., storeName=b35207b85288b14a567277d6fef4534f/B, priority=13, startTime=1732301860303; duration=0sec 2024-11-22T18:57:40,740 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:57:40,740 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b35207b85288b14a567277d6fef4534f:B 2024-11-22T18:57:40,740 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T18:57:40,741 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b35207b85288b14a567277d6fef4534f/A of b35207b85288b14a567277d6fef4534f into 495816b50c274d54b540ded95179125e(size=12.3 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:57:40,741 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b35207b85288b14a567277d6fef4534f: 2024-11-22T18:57:40,741 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., storeName=b35207b85288b14a567277d6fef4534f/A, priority=13, startTime=1732301860303; duration=0sec 2024-11-22T18:57:40,741 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:57:40,741 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b35207b85288b14a567277d6fef4534f:A 2024-11-22T18:57:40,742 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T18:57:40,742 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1541): b35207b85288b14a567277d6fef4534f/C is initiating minor compaction (all files) 2024-11-22T18:57:40,742 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b35207b85288b14a567277d6fef4534f/C in TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:40,742 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/cf6c3a6fd0214c53aa6d40627b04cd6b, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/74c0e30478f64fe3a5b0bac7d175fca1, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/18aa90f3e39947b58cc653b3acbae2df, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/418db2535f394a099c0b45e149f2f446] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp, totalSize=47.8 K 2024-11-22T18:57:40,743 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting cf6c3a6fd0214c53aa6d40627b04cd6b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1732301857144 2024-11-22T18:57:40,743 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 74c0e30478f64fe3a5b0bac7d175fca1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732301857196 2024-11-22T18:57:40,743 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 18aa90f3e39947b58cc653b3acbae2df, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1732301858393 2024-11-22T18:57:40,743 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 418db2535f394a099c0b45e149f2f446, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732301859099 2024-11-22T18:57:40,751 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b35207b85288b14a567277d6fef4534f#C#compaction#374 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:57:40,752 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/C/e4c19b0530c94d569aaa16413d06574b is 50, key is test_row_0/C:col10/1732301859109/Put/seqid=0 2024-11-22T18:57:40,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742262_1438 (size=12629) 2024-11-22T18:57:40,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-22T18:57:40,892 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-22T18:57:40,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:40,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2603): Flush status journal for b35207b85288b14a567277d6fef4534f: 2024-11-22T18:57:40,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:40,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=142 2024-11-22T18:57:40,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4169): Remote procedure done, pid=142 2024-11-22T18:57:40,896 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-22T18:57:40,896 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 155 msec 2024-11-22T18:57:40,897 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees in 160 msec 2024-11-22T18:57:41,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-22T18:57:41,056 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T18:57:41,056 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T18:57:41,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees 2024-11-22T18:57:41,058 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=143, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T18:57:41,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-22T18:57:41,059 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=143, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T18:57:41,059 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T18:57:41,164 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/C/e4c19b0530c94d569aaa16413d06574b as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/e4c19b0530c94d569aaa16413d06574b 2024-11-22T18:57:41,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-22T18:57:41,169 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b35207b85288b14a567277d6fef4534f/C of b35207b85288b14a567277d6fef4534f into e4c19b0530c94d569aaa16413d06574b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:57:41,169 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b35207b85288b14a567277d6fef4534f: 2024-11-22T18:57:41,169 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., storeName=b35207b85288b14a567277d6fef4534f/C, priority=12, startTime=1732301860303; duration=0sec 2024-11-22T18:57:41,169 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:57:41,169 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b35207b85288b14a567277d6fef4534f:C 2024-11-22T18:57:41,211 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-22T18:57:41,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:41,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2603): Flush status journal for b35207b85288b14a567277d6fef4534f: 2024-11-22T18:57:41,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:41,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=144 2024-11-22T18:57:41,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4169): Remote procedure done, pid=144 2024-11-22T18:57:41,215 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-22T18:57:41,215 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 154 msec 2024-11-22T18:57:41,217 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees in 160 msec 2024-11-22T18:57:41,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on b35207b85288b14a567277d6fef4534f 2024-11-22T18:57:41,317 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T18:57:41,317 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b35207b85288b14a567277d6fef4534f 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-22T18:57:41,322 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/A/ae0c04f92cac4e37bb7265e6103c77da is 50, key is test_row_0/A:col10/1732301861316/Put/seqid=0 2024-11-22T18:57:41,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742263_1439 (size=12151) 2024-11-22T18:57:41,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-22T18:57:41,376 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T18:57:41,376 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T18:57:41,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=145, table=TestAcidGuarantees 2024-11-22T18:57:41,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-22T18:57:41,377 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=145, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=145, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T18:57:41,378 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=145, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=145, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T18:57:41,378 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T18:57:41,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:41,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1732301921383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:41,395 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:41,395 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:41,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:41,395 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:41,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35008 deadline: 1732301921383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:41,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:41,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:41,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:41,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:41,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1732301921384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:41,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:41,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:41,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:41,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:41,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732301921426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:41,433 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to d79ba0c344fb,43593,1732301724173 for 'test_row_0' in b35207b85288b14a567277d6fef4534f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] ... 32 more 2024-11-22T18:57:41,433 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:41,433 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:41,433 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:41,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-22T18:57:41,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:41,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1732301921506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:41,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:41,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35008 deadline: 1732301921506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:41,514 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:41,514 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:41,514 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:41,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:41,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:41,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:41,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1732301921506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:41,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:41,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:41,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:41,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:41,530 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-22T18:57:41,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:41,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. as already flushing 2024-11-22T18:57:41,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:41,530 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:41,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:41,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:41,683 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-22T18:57:41,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:41,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. as already flushing 2024-11-22T18:57:41,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:41,683 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:41,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:41,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:41,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-22T18:57:41,726 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/A/ae0c04f92cac4e37bb7265e6103c77da 2024-11-22T18:57:41,733 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/B/f1681babb49648b5ade66ce00dae8f74 is 50, key is test_row_0/B:col10/1732301861316/Put/seqid=0 2024-11-22T18:57:41,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:41,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1732301921726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:41,734 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:41,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:41,734 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:41,734 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:41,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35008 deadline: 1732301921726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:41,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:41,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1732301921726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:41,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:41,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:41,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:41,735 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:41,735 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:41,735 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:41,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742264_1440 (size=12151) 2024-11-22T18:57:41,836 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-22T18:57:41,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:41,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. as already flushing 2024-11-22T18:57:41,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:41,836 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:41,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:41,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:41,989 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-22T18:57:41,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:41,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. as already flushing 2024-11-22T18:57:41,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:41,990 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:41,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:41,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:42,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-22T18:57:42,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:42,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35018 deadline: 1732301922025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:42,026 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to d79ba0c344fb,43593,1732301724173 for 'test_row_2' in b35207b85288b14a567277d6fef4534f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8251 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] ... 32 more 2024-11-22T18:57:42,026 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:42,026 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:42,026 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:42,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:42,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1732301922045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:42,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:42,050 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:42,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35008 deadline: 1732301922046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:42,050 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:42,050 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:42,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:42,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1732301922046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:42,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:42,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:42,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:42,051 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:42,051 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:42,051 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:42,137 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/B/f1681babb49648b5ade66ce00dae8f74 2024-11-22T18:57:42,142 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-22T18:57:42,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:42,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. as already flushing 2024-11-22T18:57:42,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:42,143 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:42,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:42,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:42,144 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/C/ac5093402fad45819f38fc57dc366b17 is 50, key is test_row_0/C:col10/1732301861316/Put/seqid=0 2024-11-22T18:57:42,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742265_1441 (size=12151) 2024-11-22T18:57:42,295 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-22T18:57:42,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:42,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. as already flushing 2024-11-22T18:57:42,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:42,296 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:42,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:42,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:42,448 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-22T18:57:42,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:42,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. as already flushing 2024-11-22T18:57:42,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:42,449 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:42,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:42,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:42,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-22T18:57:42,548 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/C/ac5093402fad45819f38fc57dc366b17 2024-11-22T18:57:42,552 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/A/ae0c04f92cac4e37bb7265e6103c77da as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/ae0c04f92cac4e37bb7265e6103c77da 2024-11-22T18:57:42,554 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/ae0c04f92cac4e37bb7265e6103c77da, entries=150, sequenceid=222, filesize=11.9 K 2024-11-22T18:57:42,555 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/B/f1681babb49648b5ade66ce00dae8f74 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/f1681babb49648b5ade66ce00dae8f74 2024-11-22T18:57:42,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:42,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1732301922556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:42,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:42,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35008 deadline: 1732301922556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:42,560 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:42,560 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:42,560 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:42,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:42,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:42,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:42,560 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/f1681babb49648b5ade66ce00dae8f74, entries=150, sequenceid=222, filesize=11.9 K 2024-11-22T18:57:42,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/C/ac5093402fad45819f38fc57dc366b17 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/ac5093402fad45819f38fc57dc366b17 2024-11-22T18:57:42,564 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/ac5093402fad45819f38fc57dc366b17, entries=150, sequenceid=222, filesize=11.9 K 2024-11-22T18:57:42,565 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for b35207b85288b14a567277d6fef4534f in 1248ms, sequenceid=222, compaction requested=true 2024-11-22T18:57:42,565 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b35207b85288b14a567277d6fef4534f: 2024-11-22T18:57:42,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b35207b85288b14a567277d6fef4534f:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T18:57:42,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:57:42,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b35207b85288b14a567277d6fef4534f:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T18:57:42,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:57:42,565 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T18:57:42,565 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T18:57:42,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b35207b85288b14a567277d6fef4534f:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T18:57:42,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:57:42,566 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T18:57:42,566 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1541): b35207b85288b14a567277d6fef4534f/B is initiating minor compaction (all files) 2024-11-22T18:57:42,566 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b35207b85288b14a567277d6fef4534f/B in TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:42,566 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/e39c40b8e4684fbca85fb263ec7d5f96, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/924563c13d0549c4a6015887cac32305, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/f1681babb49648b5ade66ce00dae8f74] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp, totalSize=36.0 K 2024-11-22T18:57:42,566 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T18:57:42,566 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1541): b35207b85288b14a567277d6fef4534f/A is initiating minor compaction (all files) 2024-11-22T18:57:42,566 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting e39c40b8e4684fbca85fb263ec7d5f96, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1732301858393 2024-11-22T18:57:42,566 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b35207b85288b14a567277d6fef4534f/A in TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:42,566 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/495816b50c274d54b540ded95179125e, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/6ff02883b40549b5a870ad8ddaed5b16, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/ae0c04f92cac4e37bb7265e6103c77da] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp, totalSize=36.0 K 2024-11-22T18:57:42,567 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 924563c13d0549c4a6015887cac32305, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732301859099 2024-11-22T18:57:42,567 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 495816b50c274d54b540ded95179125e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1732301858393 2024-11-22T18:57:42,567 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting f1681babb49648b5ade66ce00dae8f74, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1732301861307 2024-11-22T18:57:42,567 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ff02883b40549b5a870ad8ddaed5b16, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732301859099 2024-11-22T18:57:42,568 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae0c04f92cac4e37bb7265e6103c77da, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1732301861307 2024-11-22T18:57:42,571 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T18:57:42,571 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b35207b85288b14a567277d6fef4534f 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-22T18:57:42,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on b35207b85288b14a567277d6fef4534f 2024-11-22T18:57:42,575 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b35207b85288b14a567277d6fef4534f#B#compaction#378 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:57:42,576 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/B/319085bf5abb4c38912ca9f0fb08783d is 50, key is test_row_0/B:col10/1732301861316/Put/seqid=0 2024-11-22T18:57:42,583 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b35207b85288b14a567277d6fef4534f#A#compaction#379 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:57:42,584 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/A/bc8c28f1285242df90d34e0b4c72e71e is 50, key is test_row_0/A:col10/1732301861316/Put/seqid=0 2024-11-22T18:57:42,586 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/A/62a182371b494d00a00f405bab8a0c1e is 50, key is test_row_0/A:col10/1732301861380/Put/seqid=0 2024-11-22T18:57:42,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742266_1442 (size=12697) 2024-11-22T18:57:42,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742267_1443 (size=14541) 2024-11-22T18:57:42,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742268_1444 (size=12697) 2024-11-22T18:57:42,600 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/A/bc8c28f1285242df90d34e0b4c72e71e as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/bc8c28f1285242df90d34e0b4c72e71e 2024-11-22T18:57:42,604 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b35207b85288b14a567277d6fef4534f/A of b35207b85288b14a567277d6fef4534f into bc8c28f1285242df90d34e0b4c72e71e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:57:42,604 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-22T18:57:42,604 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b35207b85288b14a567277d6fef4534f: 2024-11-22T18:57:42,604 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., storeName=b35207b85288b14a567277d6fef4534f/A, priority=13, startTime=1732301862565; duration=0sec 2024-11-22T18:57:42,604 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:57:42,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:42,604 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b35207b85288b14a567277d6fef4534f:A 2024-11-22T18:57:42,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. as already flushing 2024-11-22T18:57:42,604 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-22T18:57:42,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:42,604 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:42,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:42,605 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-22T18:57:42,605 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-22T18:57:42,605 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. because compaction request was cancelled 2024-11-22T18:57:42,605 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b35207b85288b14a567277d6fef4534f:C 2024-11-22T18:57:42,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:42,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:42,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1732301922632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:42,644 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:42,644 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:42,644 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:42,758 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-22T18:57:42,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:42,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. as already flushing 2024-11-22T18:57:42,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:42,758 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:42,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:42,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:42,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:42,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1732301922756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:42,767 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:42,767 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:42,767 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:42,911 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-22T18:57:42,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:42,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. as already flushing 2024-11-22T18:57:42,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:42,911 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:42,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:42,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:42,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:42,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1732301922975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:42,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:42,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:42,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:42,995 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/A/62a182371b494d00a00f405bab8a0c1e 2024-11-22T18:57:43,001 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/B/319085bf5abb4c38912ca9f0fb08783d as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/319085bf5abb4c38912ca9f0fb08783d 2024-11-22T18:57:43,003 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/B/2507c6661f9e4b3281104ed1b9d92d37 is 50, key is test_row_0/B:col10/1732301861380/Put/seqid=0 2024-11-22T18:57:43,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742269_1445 (size=12151) 2024-11-22T18:57:43,009 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/B/2507c6661f9e4b3281104ed1b9d92d37 2024-11-22T18:57:43,011 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b35207b85288b14a567277d6fef4534f/B of b35207b85288b14a567277d6fef4534f into 319085bf5abb4c38912ca9f0fb08783d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:57:43,011 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b35207b85288b14a567277d6fef4534f: 2024-11-22T18:57:43,011 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., storeName=b35207b85288b14a567277d6fef4534f/B, priority=13, startTime=1732301862565; duration=0sec 2024-11-22T18:57:43,011 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:57:43,011 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b35207b85288b14a567277d6fef4534f:B 2024-11-22T18:57:43,017 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/C/4fbcdd0bf08e452297c6193bbd168d7c is 50, key is test_row_0/C:col10/1732301861380/Put/seqid=0 2024-11-22T18:57:43,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742270_1446 (size=12151) 2024-11-22T18:57:43,065 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-22T18:57:43,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:43,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. as already flushing 2024-11-22T18:57:43,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:43,065 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:43,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:43,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:43,218 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-22T18:57:43,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:43,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. as already flushing 2024-11-22T18:57:43,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:43,219 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:43,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:43,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:43,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:43,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1732301923296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:43,305 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:43,305 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:43,305 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:43,371 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-22T18:57:43,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:43,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. as already flushing 2024-11-22T18:57:43,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:43,372 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:43,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:43,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:43,422 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/C/4fbcdd0bf08e452297c6193bbd168d7c 2024-11-22T18:57:43,426 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/A/62a182371b494d00a00f405bab8a0c1e as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/62a182371b494d00a00f405bab8a0c1e 2024-11-22T18:57:43,431 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/62a182371b494d00a00f405bab8a0c1e, entries=200, sequenceid=248, filesize=14.2 K 2024-11-22T18:57:43,432 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/B/2507c6661f9e4b3281104ed1b9d92d37 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/2507c6661f9e4b3281104ed1b9d92d37 2024-11-22T18:57:43,435 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/2507c6661f9e4b3281104ed1b9d92d37, entries=150, sequenceid=248, filesize=11.9 K 2024-11-22T18:57:43,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/C/4fbcdd0bf08e452297c6193bbd168d7c as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/4fbcdd0bf08e452297c6193bbd168d7c 2024-11-22T18:57:43,440 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/4fbcdd0bf08e452297c6193bbd168d7c, entries=150, sequenceid=248, filesize=11.9 K 2024-11-22T18:57:43,441 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for b35207b85288b14a567277d6fef4534f in 870ms, sequenceid=248, compaction requested=true 2024-11-22T18:57:43,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b35207b85288b14a567277d6fef4534f: 2024-11-22T18:57:43,441 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-22T18:57:43,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b35207b85288b14a567277d6fef4534f:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T18:57:43,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:57:43,441 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-22T18:57:43,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b35207b85288b14a567277d6fef4534f:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T18:57:43,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:57:43,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b35207b85288b14a567277d6fef4534f:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T18:57:43,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:57:43,442 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-22T18:57:43,442 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-22T18:57:43,442 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. because compaction request was cancelled 2024-11-22T18:57:43,442 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b35207b85288b14a567277d6fef4534f:A 2024-11-22T18:57:43,442 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T18:57:43,442 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-22T18:57:43,442 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-22T18:57:43,442 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. because compaction request was cancelled 2024-11-22T18:57:43,442 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b35207b85288b14a567277d6fef4534f:B 2024-11-22T18:57:43,443 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T18:57:43,443 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1541): b35207b85288b14a567277d6fef4534f/C is initiating minor compaction (all files) 2024-11-22T18:57:43,443 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b35207b85288b14a567277d6fef4534f/C in TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:43,443 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/e4c19b0530c94d569aaa16413d06574b, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/ac5093402fad45819f38fc57dc366b17, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/4fbcdd0bf08e452297c6193bbd168d7c] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp, totalSize=36.1 K 2024-11-22T18:57:43,443 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4c19b0530c94d569aaa16413d06574b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732301859099 2024-11-22T18:57:43,444 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac5093402fad45819f38fc57dc366b17, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1732301861307 2024-11-22T18:57:43,444 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4fbcdd0bf08e452297c6193bbd168d7c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732301861380 2024-11-22T18:57:43,454 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b35207b85288b14a567277d6fef4534f#C#compaction#383 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:57:43,455 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/C/4dcad8574a9940c6b8f54b61a1f95cc6 is 50, key is test_row_0/C:col10/1732301861380/Put/seqid=0 2024-11-22T18:57:43,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742271_1447 (size=12731) 2024-11-22T18:57:43,525 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-22T18:57:43,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:43,525 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2902): Flushing b35207b85288b14a567277d6fef4534f 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-22T18:57:43,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-22T18:57:43,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/A/1bad2857105246e98935398e1af0ba7f is 50, key is test_row_0/A:col10/1732301862619/Put/seqid=0 2024-11-22T18:57:43,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742272_1448 (size=12251) 2024-11-22T18:57:43,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. as already flushing 2024-11-22T18:57:43,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on b35207b85288b14a567277d6fef4534f 2024-11-22T18:57:43,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:43,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35008 deadline: 1732301923659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:43,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:43,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1732301923661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:43,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:43,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:43,672 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:43,672 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:43,672 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:43,672 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:43,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:43,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35008 deadline: 1732301923776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:43,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:43,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:43,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:43,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:43,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1732301923776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:43,779 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:43,779 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:43,779 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:43,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:43,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1732301923815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:43,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:43,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:43,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:43,870 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/C/4dcad8574a9940c6b8f54b61a1f95cc6 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/4dcad8574a9940c6b8f54b61a1f95cc6 2024-11-22T18:57:43,874 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b35207b85288b14a567277d6fef4534f/C of b35207b85288b14a567277d6fef4534f into 4dcad8574a9940c6b8f54b61a1f95cc6(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:57:43,874 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b35207b85288b14a567277d6fef4534f: 2024-11-22T18:57:43,875 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., storeName=b35207b85288b14a567277d6fef4534f/C, priority=13, startTime=1732301863441; duration=0sec 2024-11-22T18:57:43,875 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:57:43,875 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b35207b85288b14a567277d6fef4534f:C 2024-11-22T18:57:43,935 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/A/1bad2857105246e98935398e1af0ba7f 2024-11-22T18:57:43,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/B/e8fc86d75ba2431784d1e169e031076c is 50, key is test_row_0/B:col10/1732301862619/Put/seqid=0 2024-11-22T18:57:43,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742273_1449 (size=12251) 2024-11-22T18:57:43,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:43,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35008 deadline: 1732301923985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:43,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:43,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:43,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:43,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:43,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1732301923986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:43,997 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:43,997 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:43,997 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:44,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:44,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35008 deadline: 1732301924296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:44,304 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:44,304 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:44,304 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:44,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:44,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1732301924306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:44,316 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:44,316 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:44,316 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:44,346 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/B/e8fc86d75ba2431784d1e169e031076c 2024-11-22T18:57:44,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/C/c5599aaeca7347e4ae3fa171bc3d849c is 50, key is test_row_0/C:col10/1732301862619/Put/seqid=0 2024-11-22T18:57:44,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742274_1450 (size=12251) 2024-11-22T18:57:44,758 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/C/c5599aaeca7347e4ae3fa171bc3d849c 2024-11-22T18:57:44,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/A/1bad2857105246e98935398e1af0ba7f as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/1bad2857105246e98935398e1af0ba7f 2024-11-22T18:57:44,765 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/1bad2857105246e98935398e1af0ba7f, entries=150, sequenceid=260, filesize=12.0 K 2024-11-22T18:57:44,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/B/e8fc86d75ba2431784d1e169e031076c as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/e8fc86d75ba2431784d1e169e031076c 2024-11-22T18:57:44,769 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/e8fc86d75ba2431784d1e169e031076c, entries=150, sequenceid=260, filesize=12.0 K 2024-11-22T18:57:44,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/C/c5599aaeca7347e4ae3fa171bc3d849c as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/c5599aaeca7347e4ae3fa171bc3d849c 2024-11-22T18:57:44,773 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/c5599aaeca7347e4ae3fa171bc3d849c, entries=150, sequenceid=260, filesize=12.0 K 2024-11-22T18:57:44,774 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for b35207b85288b14a567277d6fef4534f in 1249ms, sequenceid=260, compaction requested=true 2024-11-22T18:57:44,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2603): Flush status journal for b35207b85288b14a567277d6fef4534f: 2024-11-22T18:57:44,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:44,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=146 2024-11-22T18:57:44,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4169): Remote procedure done, pid=146 2024-11-22T18:57:44,777 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-22T18:57:44,777 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3970 sec 2024-11-22T18:57:44,778 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=145, table=TestAcidGuarantees in 3.4010 sec 2024-11-22T18:57:44,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on b35207b85288b14a567277d6fef4534f 2024-11-22T18:57:44,822 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T18:57:44,822 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b35207b85288b14a567277d6fef4534f 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-22T18:57:44,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/A/c1acf6de814b4f23827847c15c122142 is 50, key is test_row_0/A:col10/1732301864820/Put/seqid=0 2024-11-22T18:57:44,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742275_1451 (size=14741) 2024-11-22T18:57:44,830 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/A/c1acf6de814b4f23827847c15c122142 2024-11-22T18:57:44,840 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/B/92f5cca2383949128d406c045d892c40 is 50, key is test_row_0/B:col10/1732301864820/Put/seqid=0 2024-11-22T18:57:44,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742276_1452 (size=12301) 2024-11-22T18:57:44,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:44,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1732301924842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:44,848 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:44,848 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:44,849 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:44,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:44,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1732301924845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:44,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:44,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:44,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:44,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:44,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35008 deadline: 1732301924848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:44,863 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:44,863 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:44,863 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:44,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:44,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1732301924956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:44,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:44,961 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:44,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1732301924956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:44,961 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:44,961 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:44,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:44,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:44,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:44,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:44,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35008 deadline: 1732301924965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:44,973 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:44,973 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:44,973 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:45,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:45,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1732301925166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:45,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:45,174 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:45,174 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:45,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1732301925166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:45,174 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:45,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:45,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:45,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:45,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:45,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35008 deadline: 1732301925185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:45,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:45,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:45,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:45,244 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/B/92f5cca2383949128d406c045d892c40 2024-11-22T18:57:45,251 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/C/0d2849da796943f59c1d2fa148df9aa4 is 50, key is test_row_0/C:col10/1732301864820/Put/seqid=0 2024-11-22T18:57:45,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742277_1453 (size=12301) 2024-11-22T18:57:45,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:45,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35010 deadline: 1732301925475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:45,485 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to d79ba0c344fb,43593,1732301724173 for 'test_row_0' in b35207b85288b14a567277d6fef4534f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8264 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] ... 32 more 2024-11-22T18:57:45,486 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:45,486 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:45,486 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:45,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:45,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1732301925485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:45,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:45,490 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:45,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1732301925485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:45,490 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:45,490 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:45,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:45,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:45,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:45,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:45,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35008 deadline: 1732301925496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:45,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:45,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:45,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:45,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-22T18:57:45,546 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T18:57:45,546 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T18:57:45,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] procedure2.ProcedureExecutor(1139): Stored pid=147, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=147, table=TestAcidGuarantees 2024-11-22T18:57:45,548 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=147, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=147, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T18:57:45,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-22T18:57:45,549 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=147, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=147, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T18:57:45,549 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T18:57:45,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-22T18:57:45,657 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/C/0d2849da796943f59c1d2fa148df9aa4 2024-11-22T18:57:45,665 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/A/c1acf6de814b4f23827847c15c122142 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/c1acf6de814b4f23827847c15c122142 2024-11-22T18:57:45,673 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/c1acf6de814b4f23827847c15c122142, entries=200, sequenceid=288, filesize=14.4 K 2024-11-22T18:57:45,674 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/B/92f5cca2383949128d406c045d892c40 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/92f5cca2383949128d406c045d892c40 2024-11-22T18:57:45,678 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/92f5cca2383949128d406c045d892c40, entries=150, sequenceid=288, filesize=12.0 K 2024-11-22T18:57:45,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/C/0d2849da796943f59c1d2fa148df9aa4 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/0d2849da796943f59c1d2fa148df9aa4 2024-11-22T18:57:45,686 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/0d2849da796943f59c1d2fa148df9aa4, entries=150, sequenceid=288, filesize=12.0 K 2024-11-22T18:57:45,687 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for b35207b85288b14a567277d6fef4534f in 865ms, sequenceid=288, compaction requested=true 2024-11-22T18:57:45,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b35207b85288b14a567277d6fef4534f: 2024-11-22T18:57:45,688 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T18:57:45,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b35207b85288b14a567277d6fef4534f:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T18:57:45,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:57:45,688 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T18:57:45,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b35207b85288b14a567277d6fef4534f:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T18:57:45,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:57:45,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b35207b85288b14a567277d6fef4534f:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T18:57:45,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:57:45,693 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54230 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T18:57:45,693 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1541): b35207b85288b14a567277d6fef4534f/A is initiating minor compaction (all files) 2024-11-22T18:57:45,693 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b35207b85288b14a567277d6fef4534f/A in TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:45,693 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/bc8c28f1285242df90d34e0b4c72e71e, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/62a182371b494d00a00f405bab8a0c1e, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/1bad2857105246e98935398e1af0ba7f, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/c1acf6de814b4f23827847c15c122142] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp, totalSize=53.0 K 2024-11-22T18:57:45,694 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc8c28f1285242df90d34e0b4c72e71e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1732301861307 2024-11-22T18:57:45,694 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62a182371b494d00a00f405bab8a0c1e, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732301861380 2024-11-22T18:57:45,694 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1bad2857105246e98935398e1af0ba7f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1732301862616 2024-11-22T18:57:45,695 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1acf6de814b4f23827847c15c122142, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732301863650 2024-11-22T18:57:45,700 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49400 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T18:57:45,700 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1541): b35207b85288b14a567277d6fef4534f/B is initiating minor compaction (all files) 2024-11-22T18:57:45,700 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b35207b85288b14a567277d6fef4534f/B in TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:45,700 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/319085bf5abb4c38912ca9f0fb08783d, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/2507c6661f9e4b3281104ed1b9d92d37, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/e8fc86d75ba2431784d1e169e031076c, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/92f5cca2383949128d406c045d892c40] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp, totalSize=48.2 K 2024-11-22T18:57:45,700 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 319085bf5abb4c38912ca9f0fb08783d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1732301861307 2024-11-22T18:57:45,701 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 2507c6661f9e4b3281104ed1b9d92d37, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732301861380 2024-11-22T18:57:45,701 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=148 2024-11-22T18:57:45,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:45,701 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegion(2902): Flushing b35207b85288b14a567277d6fef4534f 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-22T18:57:45,702 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting e8fc86d75ba2431784d1e169e031076c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1732301862616 2024-11-22T18:57:45,703 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 92f5cca2383949128d406c045d892c40, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732301863657 2024-11-22T18:57:45,732 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b35207b85288b14a567277d6fef4534f#A#compaction#390 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:57:45,733 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/A/5adc3072171d4baf9a599de0ffa50f2c is 50, key is test_row_0/A:col10/1732301864820/Put/seqid=0 2024-11-22T18:57:45,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/A/95c3f947398549ab99f4beb95eac423e is 50, key is test_row_0/A:col10/1732301864841/Put/seqid=0 2024-11-22T18:57:45,741 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b35207b85288b14a567277d6fef4534f#B#compaction#392 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:57:45,741 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/B/8661be6db4f24ed48ad7372deb4cb05a is 50, key is test_row_0/B:col10/1732301864820/Put/seqid=0 2024-11-22T18:57:45,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742278_1454 (size=12983) 2024-11-22T18:57:45,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742280_1456 (size=12301) 2024-11-22T18:57:45,760 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/A/5adc3072171d4baf9a599de0ffa50f2c as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/5adc3072171d4baf9a599de0ffa50f2c 2024-11-22T18:57:45,761 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/A/95c3f947398549ab99f4beb95eac423e 2024-11-22T18:57:45,767 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b35207b85288b14a567277d6fef4534f/A of b35207b85288b14a567277d6fef4534f into 5adc3072171d4baf9a599de0ffa50f2c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:57:45,767 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b35207b85288b14a567277d6fef4534f: 2024-11-22T18:57:45,767 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., storeName=b35207b85288b14a567277d6fef4534f/A, priority=12, startTime=1732301865687; duration=0sec 2024-11-22T18:57:45,768 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:57:45,768 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b35207b85288b14a567277d6fef4534f:A 2024-11-22T18:57:45,768 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T18:57:45,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742279_1455 (size=12983) 2024-11-22T18:57:45,770 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37283 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T18:57:45,770 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1541): b35207b85288b14a567277d6fef4534f/C is initiating minor compaction (all files) 2024-11-22T18:57:45,770 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b35207b85288b14a567277d6fef4534f/C in TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:45,770 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/4dcad8574a9940c6b8f54b61a1f95cc6, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/c5599aaeca7347e4ae3fa171bc3d849c, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/0d2849da796943f59c1d2fa148df9aa4] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp, totalSize=36.4 K 2024-11-22T18:57:45,771 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4dcad8574a9940c6b8f54b61a1f95cc6, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732301861380 2024-11-22T18:57:45,773 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5599aaeca7347e4ae3fa171bc3d849c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1732301862616 2024-11-22T18:57:45,775 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d2849da796943f59c1d2fa148df9aa4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732301863657 2024-11-22T18:57:45,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/B/07a8d0e21f944201b182fa68eea42152 is 50, key is test_row_0/B:col10/1732301864841/Put/seqid=0 2024-11-22T18:57:45,779 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/B/8661be6db4f24ed48ad7372deb4cb05a as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/8661be6db4f24ed48ad7372deb4cb05a 2024-11-22T18:57:45,788 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b35207b85288b14a567277d6fef4534f/B of b35207b85288b14a567277d6fef4534f into 8661be6db4f24ed48ad7372deb4cb05a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:57:45,788 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b35207b85288b14a567277d6fef4534f: 2024-11-22T18:57:45,788 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., storeName=b35207b85288b14a567277d6fef4534f/B, priority=12, startTime=1732301865688; duration=0sec 2024-11-22T18:57:45,788 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:57:45,788 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b35207b85288b14a567277d6fef4534f:B 2024-11-22T18:57:45,795 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b35207b85288b14a567277d6fef4534f#C#compaction#394 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:57:45,795 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/C/e8e5840b2ef6470b887485de3693bf30 is 50, key is test_row_0/C:col10/1732301864820/Put/seqid=0 2024-11-22T18:57:45,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742281_1457 (size=12301) 2024-11-22T18:57:45,799 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/B/07a8d0e21f944201b182fa68eea42152 2024-11-22T18:57:45,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/C/1268a636e30b4b62b7639d24c2237df8 is 50, key is test_row_0/C:col10/1732301864841/Put/seqid=0 2024-11-22T18:57:45,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742282_1458 (size=12983) 2024-11-22T18:57:45,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742283_1459 (size=12301) 2024-11-22T18:57:45,817 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/C/e8e5840b2ef6470b887485de3693bf30 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/e8e5840b2ef6470b887485de3693bf30 2024-11-22T18:57:45,820 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b35207b85288b14a567277d6fef4534f/C of b35207b85288b14a567277d6fef4534f into e8e5840b2ef6470b887485de3693bf30(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:57:45,821 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b35207b85288b14a567277d6fef4534f: 2024-11-22T18:57:45,821 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., storeName=b35207b85288b14a567277d6fef4534f/C, priority=13, startTime=1732301865688; duration=0sec 2024-11-22T18:57:45,821 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:57:45,821 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b35207b85288b14a567277d6fef4534f:C 2024-11-22T18:57:45,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-22T18:57:46,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on b35207b85288b14a567277d6fef4534f 2024-11-22T18:57:46,000 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. as already flushing 2024-11-22T18:57:46,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:46,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1732301926058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:46,068 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:46,068 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:46,068 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:46,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:46,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35008 deadline: 1732301926066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:46,069 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:46,069 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:46,069 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:46,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:46,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1732301926067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:46,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:46,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:46,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:46,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-22T18:57:46,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:46,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1732301926176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:46,186 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:46,186 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:46,186 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:46,186 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:46,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35008 deadline: 1732301926176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:46,186 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:46,186 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:46,186 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:46,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:46,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1732301926185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:46,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:46,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:46,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:46,211 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/C/1268a636e30b4b62b7639d24c2237df8 2024-11-22T18:57:46,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/A/95c3f947398549ab99f4beb95eac423e as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/95c3f947398549ab99f4beb95eac423e 2024-11-22T18:57:46,220 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/95c3f947398549ab99f4beb95eac423e, entries=150, sequenceid=297, filesize=12.0 K 2024-11-22T18:57:46,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/B/07a8d0e21f944201b182fa68eea42152 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/07a8d0e21f944201b182fa68eea42152 2024-11-22T18:57:46,225 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/07a8d0e21f944201b182fa68eea42152, entries=150, sequenceid=297, filesize=12.0 K 2024-11-22T18:57:46,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/C/1268a636e30b4b62b7639d24c2237df8 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/1268a636e30b4b62b7639d24c2237df8 2024-11-22T18:57:46,229 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/1268a636e30b4b62b7639d24c2237df8, entries=150, sequenceid=297, filesize=12.0 K 2024-11-22T18:57:46,230 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for b35207b85288b14a567277d6fef4534f in 529ms, sequenceid=297, compaction requested=false 2024-11-22T18:57:46,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegion(2603): Flush status journal for b35207b85288b14a567277d6fef4534f: 2024-11-22T18:57:46,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:46,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=148 2024-11-22T18:57:46,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4169): Remote procedure done, pid=148 2024-11-22T18:57:46,233 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-22T18:57:46,233 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 683 msec 2024-11-22T18:57:46,235 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=147, table=TestAcidGuarantees in 688 msec 2024-11-22T18:57:46,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on b35207b85288b14a567277d6fef4534f 2024-11-22T18:57:46,405 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T18:57:46,405 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b35207b85288b14a567277d6fef4534f 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-22T18:57:46,411 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/A/e34eff47556747089ba08f99671c37ee is 50, key is test_row_0/A:col10/1732301866065/Put/seqid=0 2024-11-22T18:57:46,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742284_1460 (size=17181) 2024-11-22T18:57:46,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:46,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1732301926415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:46,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:46,425 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:46,425 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:46,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35008 deadline: 1732301926416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:46,425 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:46,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:46,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1732301926416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:46,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:46,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:46,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:46,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:46,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:46,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:46,542 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:46,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1732301926535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:46,542 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:46,542 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:46,542 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:46,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1732301926535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:46,542 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:46,542 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:46,542 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:46,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35008 deadline: 1732301926535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:46,543 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:46,543 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:46,543 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:46,543 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:46,543 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:46,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-22T18:57:46,686 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T18:57:46,686 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T18:57:46,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] procedure2.ProcedureExecutor(1139): Stored pid=149, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=149, table=TestAcidGuarantees 2024-11-22T18:57:46,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-11-22T18:57:46,688 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=149, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=149, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T18:57:46,689 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=149, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=149, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T18:57:46,689 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T18:57:46,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:46,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1732301926745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:46,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:46,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:46,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:46,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1732301926745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:46,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:46,753 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:46,753 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:46,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:46,753 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:46,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35008 deadline: 1732301926745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:46,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:46,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:46,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:46,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-11-22T18:57:46,815 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/A/e34eff47556747089ba08f99671c37ee 2024-11-22T18:57:46,823 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/B/02f3d7a622314f37928b500c7507a6ec is 50, key is test_row_0/B:col10/1732301866065/Put/seqid=0 2024-11-22T18:57:46,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742285_1461 (size=12301) 2024-11-22T18:57:46,841 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=150 2024-11-22T18:57:46,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:46,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. as already flushing 2024-11-22T18:57:46,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:46,841 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=150}] handler.RSProcedureHandler(58): pid=150 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:46,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=150 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:46,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=150 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:46,994 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=150 2024-11-22T18:57:46,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:46,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. as already flushing 2024-11-22T18:57:46,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:46,994 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=150}] handler.RSProcedureHandler(58): pid=150 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:46,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=150 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:46,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=150 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:47,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-11-22T18:57:47,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:47,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1732301927056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:47,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:47,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1732301927056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:47,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:47,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:47,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:47,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:47,059 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:47,059 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:47,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35008 deadline: 1732301927056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:47,060 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:47,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:47,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:47,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:47,147 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=150 2024-11-22T18:57:47,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:47,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. as already flushing 2024-11-22T18:57:47,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:47,148 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] handler.RSProcedureHandler(58): pid=150 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:47,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=150 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:47,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=150 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:47,227 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/B/02f3d7a622314f37928b500c7507a6ec 2024-11-22T18:57:47,234 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/C/254b16d6d9d441028a7e28285de4893f is 50, key is test_row_0/C:col10/1732301866065/Put/seqid=0 2024-11-22T18:57:47,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742286_1462 (size=12301) 2024-11-22T18:57:47,245 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/C/254b16d6d9d441028a7e28285de4893f 2024-11-22T18:57:47,248 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/A/e34eff47556747089ba08f99671c37ee as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/e34eff47556747089ba08f99671c37ee 2024-11-22T18:57:47,251 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/e34eff47556747089ba08f99671c37ee, entries=250, sequenceid=329, filesize=16.8 K 2024-11-22T18:57:47,252 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/B/02f3d7a622314f37928b500c7507a6ec as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/02f3d7a622314f37928b500c7507a6ec 2024-11-22T18:57:47,255 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/02f3d7a622314f37928b500c7507a6ec, entries=150, sequenceid=329, filesize=12.0 K 2024-11-22T18:57:47,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/C/254b16d6d9d441028a7e28285de4893f as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/254b16d6d9d441028a7e28285de4893f 2024-11-22T18:57:47,260 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/254b16d6d9d441028a7e28285de4893f, entries=150, sequenceid=329, filesize=12.0 K 2024-11-22T18:57:47,261 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=26.84 KB/27480 for b35207b85288b14a567277d6fef4534f in 855ms, sequenceid=329, compaction requested=true 2024-11-22T18:57:47,261 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b35207b85288b14a567277d6fef4534f: 2024-11-22T18:57:47,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b35207b85288b14a567277d6fef4534f:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T18:57:47,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:57:47,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b35207b85288b14a567277d6fef4534f:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T18:57:47,261 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T18:57:47,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:57:47,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b35207b85288b14a567277d6fef4534f:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T18:57:47,261 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T18:57:47,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:57:47,262 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42465 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T18:57:47,262 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1541): b35207b85288b14a567277d6fef4534f/A is initiating minor compaction (all files) 2024-11-22T18:57:47,262 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b35207b85288b14a567277d6fef4534f/A in TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:47,262 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/5adc3072171d4baf9a599de0ffa50f2c, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/95c3f947398549ab99f4beb95eac423e, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/e34eff47556747089ba08f99671c37ee] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp, totalSize=41.5 K 2024-11-22T18:57:47,263 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T18:57:47,263 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1541): b35207b85288b14a567277d6fef4534f/B is initiating minor compaction (all files) 2024-11-22T18:57:47,263 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b35207b85288b14a567277d6fef4534f/B in TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:47,264 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/8661be6db4f24ed48ad7372deb4cb05a, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/07a8d0e21f944201b182fa68eea42152, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/02f3d7a622314f37928b500c7507a6ec] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp, totalSize=36.7 K 2024-11-22T18:57:47,264 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5adc3072171d4baf9a599de0ffa50f2c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732301863657 2024-11-22T18:57:47,264 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 8661be6db4f24ed48ad7372deb4cb05a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732301863657 2024-11-22T18:57:47,264 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95c3f947398549ab99f4beb95eac423e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1732301864835 2024-11-22T18:57:47,264 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 07a8d0e21f944201b182fa68eea42152, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1732301864835 2024-11-22T18:57:47,264 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting e34eff47556747089ba08f99671c37ee, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1732301866055 2024-11-22T18:57:47,264 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 02f3d7a622314f37928b500c7507a6ec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1732301866055 2024-11-22T18:57:47,272 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b35207b85288b14a567277d6fef4534f#B#compaction#399 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:57:47,273 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/B/b4875a75f0dd4deaa620ffd561beaefe is 50, key is test_row_0/B:col10/1732301866065/Put/seqid=0 2024-11-22T18:57:47,278 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b35207b85288b14a567277d6fef4534f#A#compaction#400 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:57:47,278 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/A/5abe02d0bb8042bfb74837edc9376b2a is 50, key is test_row_0/A:col10/1732301866065/Put/seqid=0 2024-11-22T18:57:47,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742287_1463 (size=13085) 2024-11-22T18:57:47,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742288_1464 (size=13085) 2024-11-22T18:57:47,300 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=150 2024-11-22T18:57:47,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:47,300 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.HRegion(2902): Flushing b35207b85288b14a567277d6fef4534f 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-22T18:57:47,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=150}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/A/10e56784fb4a4ca39cb53a0a8ddf3888 is 50, key is test_row_0/A:col10/1732301866415/Put/seqid=0 2024-11-22T18:57:47,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742289_1465 (size=12301) 2024-11-22T18:57:47,311 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/A/10e56784fb4a4ca39cb53a0a8ddf3888 2024-11-22T18:57:47,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-11-22T18:57:47,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=150}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/B/ddef69eb82e34182ab58deab34e41a4d is 50, key is test_row_0/B:col10/1732301866415/Put/seqid=0 2024-11-22T18:57:47,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742290_1466 (size=12301) 2024-11-22T18:57:47,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on b35207b85288b14a567277d6fef4534f 2024-11-22T18:57:47,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. as already flushing 2024-11-22T18:57:47,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:47,661 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:47,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35008 deadline: 1732301927646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:47,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1732301927647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:47,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:47,661 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:47,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:47,661 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:47,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:47,661 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:47,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:47,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1732301927662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:47,675 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:47,675 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:47,675 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:47,691 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/A/5abe02d0bb8042bfb74837edc9376b2a as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/5abe02d0bb8042bfb74837edc9376b2a 2024-11-22T18:57:47,691 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/B/b4875a75f0dd4deaa620ffd561beaefe as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/b4875a75f0dd4deaa620ffd561beaefe 2024-11-22T18:57:47,695 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b35207b85288b14a567277d6fef4534f/A of b35207b85288b14a567277d6fef4534f into 5abe02d0bb8042bfb74837edc9376b2a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:57:47,695 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b35207b85288b14a567277d6fef4534f: 2024-11-22T18:57:47,695 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., storeName=b35207b85288b14a567277d6fef4534f/A, priority=13, startTime=1732301867261; duration=0sec 2024-11-22T18:57:47,695 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:57:47,695 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b35207b85288b14a567277d6fef4534f:A 2024-11-22T18:57:47,695 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T18:57:47,696 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b35207b85288b14a567277d6fef4534f/B of b35207b85288b14a567277d6fef4534f into b4875a75f0dd4deaa620ffd561beaefe(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:57:47,696 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b35207b85288b14a567277d6fef4534f: 2024-11-22T18:57:47,696 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., storeName=b35207b85288b14a567277d6fef4534f/B, priority=13, startTime=1732301867261; duration=0sec 2024-11-22T18:57:47,696 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T18:57:47,696 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1541): b35207b85288b14a567277d6fef4534f/C is initiating minor compaction (all files) 2024-11-22T18:57:47,696 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b35207b85288b14a567277d6fef4534f/C in TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:47,696 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:57:47,696 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/e8e5840b2ef6470b887485de3693bf30, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/1268a636e30b4b62b7639d24c2237df8, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/254b16d6d9d441028a7e28285de4893f] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp, totalSize=36.7 K 2024-11-22T18:57:47,696 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b35207b85288b14a567277d6fef4534f:B 2024-11-22T18:57:47,697 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8e5840b2ef6470b887485de3693bf30, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732301863657 2024-11-22T18:57:47,698 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1268a636e30b4b62b7639d24c2237df8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1732301864835 2024-11-22T18:57:47,698 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 254b16d6d9d441028a7e28285de4893f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1732301866055 2024-11-22T18:57:47,703 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b35207b85288b14a567277d6fef4534f#C#compaction#403 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:57:47,704 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/C/87d18eea275942c7ae14c2edbb11974a is 50, key is test_row_0/C:col10/1732301866065/Put/seqid=0 2024-11-22T18:57:47,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742291_1467 (size=13085) 2024-11-22T18:57:47,734 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/B/ddef69eb82e34182ab58deab34e41a4d 2024-11-22T18:57:47,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=150}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/C/3e8d196976f14b6fa938ff1619ce74ae is 50, key is test_row_0/C:col10/1732301866415/Put/seqid=0 2024-11-22T18:57:47,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742292_1468 (size=12301) 2024-11-22T18:57:47,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:47,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35008 deadline: 1732301927765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:47,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:47,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:47,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:47,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:47,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1732301927765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:47,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:47,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:47,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:47,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:47,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1732301927785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:47,794 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:47,794 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:47,794 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:47,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-11-22T18:57:47,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:47,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35008 deadline: 1732301927975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:47,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:47,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:47,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:47,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:47,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1732301927985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:47,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:47,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:47,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:48,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:48,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1732301928005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:48,015 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:48,015 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:48,015 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:48,113 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/C/87d18eea275942c7ae14c2edbb11974a as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/87d18eea275942c7ae14c2edbb11974a 2024-11-22T18:57:48,117 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b35207b85288b14a567277d6fef4534f/C of b35207b85288b14a567277d6fef4534f into 87d18eea275942c7ae14c2edbb11974a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:57:48,117 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b35207b85288b14a567277d6fef4534f: 2024-11-22T18:57:48,117 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., storeName=b35207b85288b14a567277d6fef4534f/C, priority=13, startTime=1732301867261; duration=0sec 2024-11-22T18:57:48,117 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:57:48,117 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b35207b85288b14a567277d6fef4534f:C 2024-11-22T18:57:48,143 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/C/3e8d196976f14b6fa938ff1619ce74ae 2024-11-22T18:57:48,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/A/10e56784fb4a4ca39cb53a0a8ddf3888 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/10e56784fb4a4ca39cb53a0a8ddf3888 2024-11-22T18:57:48,151 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/10e56784fb4a4ca39cb53a0a8ddf3888, entries=150, sequenceid=336, filesize=12.0 K 2024-11-22T18:57:48,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/B/ddef69eb82e34182ab58deab34e41a4d as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/ddef69eb82e34182ab58deab34e41a4d 2024-11-22T18:57:48,155 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/ddef69eb82e34182ab58deab34e41a4d, entries=150, sequenceid=336, filesize=12.0 K 2024-11-22T18:57:48,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/C/3e8d196976f14b6fa938ff1619ce74ae as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/3e8d196976f14b6fa938ff1619ce74ae 2024-11-22T18:57:48,159 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/3e8d196976f14b6fa938ff1619ce74ae, entries=150, sequenceid=336, filesize=12.0 K 2024-11-22T18:57:48,160 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=174.43 KB/178620 for b35207b85288b14a567277d6fef4534f in 860ms, sequenceid=336, compaction requested=false 2024-11-22T18:57:48,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.HRegion(2603): Flush status journal for b35207b85288b14a567277d6fef4534f: 2024-11-22T18:57:48,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:48,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=150 2024-11-22T18:57:48,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4169): Remote procedure done, pid=150 2024-11-22T18:57:48,165 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149 2024-11-22T18:57:48,166 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4750 sec 2024-11-22T18:57:48,168 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=149, table=TestAcidGuarantees in 1.4800 sec 2024-11-22T18:57:48,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on b35207b85288b14a567277d6fef4534f 2024-11-22T18:57:48,294 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T18:57:48,294 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b35207b85288b14a567277d6fef4534f 3/3 column families, dataSize=181.14 KB heapSize=475.36 KB 2024-11-22T18:57:48,300 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/A/91fc73a0da4f46fea5c096f96d61f149 is 50, key is test_row_0/A:col10/1732301867643/Put/seqid=0 2024-11-22T18:57:48,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742293_1469 (size=14741) 2024-11-22T18:57:48,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:48,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1732301928305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:48,308 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:48,308 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:48,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:48,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:48,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35008 deadline: 1732301928308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:48,313 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:48,313 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:48,313 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:48,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:48,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1732301928326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:48,332 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:48,332 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:48,332 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:48,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:48,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1732301928416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:48,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:48,423 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:48,423 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:48,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35008 deadline: 1732301928416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:48,423 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:48,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:48,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:48,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:48,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:48,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1732301928627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:48,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:48,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35008 deadline: 1732301928627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:48,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:48,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:48,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:48,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:48,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:48,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:48,706 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/A/91fc73a0da4f46fea5c096f96d61f149 2024-11-22T18:57:48,715 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/B/b9110ddb2e5e42478395a478ff2c65a9 is 50, key is test_row_0/B:col10/1732301867643/Put/seqid=0 2024-11-22T18:57:48,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742294_1470 (size=12301) 2024-11-22T18:57:48,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-11-22T18:57:48,836 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T18:57:48,837 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T18:57:48,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] procedure2.ProcedureExecutor(1139): Stored pid=151, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=151, table=TestAcidGuarantees 2024-11-22T18:57:48,839 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=151, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=151, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T18:57:48,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=151 2024-11-22T18:57:48,840 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=151, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=151, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T18:57:48,840 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=152, ppid=151, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T18:57:48,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:48,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35032 deadline: 1732301928846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:48,852 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:48,852 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:48,852 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:48,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=151 2024-11-22T18:57:48,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:48,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35008 deadline: 1732301928945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:48,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:48,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1732301928947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:48,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:48,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:48,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:48,955 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:48,955 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:48,955 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:48,975 INFO [Thread-1802 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1802. 2024-11-22T18:57:48,975 DEBUG [Thread-1802 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T18:57:48,975 DEBUG [Thread-1802 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:57:48,975 DEBUG [Thread-1802 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:57:48,976 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T18:57:48,980 INFO [Thread-1796 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1796. 2024-11-22T18:57:48,980 DEBUG [Thread-1796 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T18:57:48,980 DEBUG [Thread-1796 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:57:48,980 DEBUG [Thread-1796 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:57:48,980 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T18:57:48,980 INFO [Thread-1800 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1800. 2024-11-22T18:57:48,981 DEBUG [Thread-1800 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T18:57:48,981 DEBUG [Thread-1800 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:57:48,981 DEBUG [Thread-1800 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:57:48,981 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T18:57:48,985 INFO [Thread-1794 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1794. 2024-11-22T18:57:48,985 DEBUG [Thread-1794 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T18:57:48,985 DEBUG [Thread-1794 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:57:48,985 DEBUG [Thread-1794 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:57:48,985 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T18:57:48,988 INFO [Thread-1798 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1798. 2024-11-22T18:57:48,988 DEBUG [Thread-1798 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T18:57:48,988 DEBUG [Thread-1798 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:57:48,988 DEBUG [Thread-1798 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:57:48,988 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T18:57:48,993 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=152 2024-11-22T18:57:48,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:48,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. as already flushing 2024-11-22T18:57:48,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:48,994 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=152}] handler.RSProcedureHandler(58): pid=152 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:48,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=152 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:48,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=152 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:49,120 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/B/b9110ddb2e5e42478395a478ff2c65a9 2024-11-22T18:57:49,128 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/C/813438c61e3841b782cea760d0ed120d is 50, key is test_row_0/C:col10/1732301867643/Put/seqid=0 2024-11-22T18:57:49,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742295_1471 (size=12301) 2024-11-22T18:57:49,147 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=152 2024-11-22T18:57:49,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:49,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. as already flushing 2024-11-22T18:57:49,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:49,147 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] handler.RSProcedureHandler(58): pid=152 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:49,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=152 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:49,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=152 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:49,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=151 2024-11-22T18:57:49,300 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=152 2024-11-22T18:57:49,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:49,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. as already flushing 2024-11-22T18:57:49,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:49,301 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] handler.RSProcedureHandler(58): pid=152 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:49,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=152 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:49,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=152 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:49,454 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=152 2024-11-22T18:57:49,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:49,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. as already flushing 2024-11-22T18:57:49,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:49,454 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=152}] handler.RSProcedureHandler(58): pid=152 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:49,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=152 java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:49,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=152 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:57:49,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=151 2024-11-22T18:57:49,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:49,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35058 deadline: 1732301929466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:49,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:57:49,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35008 deadline: 1732301929466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:49,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:49,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:49,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:49,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 , the old value is region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:49,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:57:49,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., hostname=d79ba0c344fb,43593,1732301724173, seqNum=2 because the exception is null or not the one we care about 2024-11-22T18:57:49,534 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/C/813438c61e3841b782cea760d0ed120d 2024-11-22T18:57:49,538 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/A/91fc73a0da4f46fea5c096f96d61f149 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/91fc73a0da4f46fea5c096f96d61f149 2024-11-22T18:57:49,541 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/91fc73a0da4f46fea5c096f96d61f149, entries=200, sequenceid=369, filesize=14.4 K 2024-11-22T18:57:49,541 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/B/b9110ddb2e5e42478395a478ff2c65a9 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/b9110ddb2e5e42478395a478ff2c65a9 2024-11-22T18:57:49,544 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/b9110ddb2e5e42478395a478ff2c65a9, entries=150, sequenceid=369, filesize=12.0 K 2024-11-22T18:57:49,545 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/C/813438c61e3841b782cea760d0ed120d as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/813438c61e3841b782cea760d0ed120d 2024-11-22T18:57:49,548 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/813438c61e3841b782cea760d0ed120d, entries=150, sequenceid=369, filesize=12.0 K 2024-11-22T18:57:49,549 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~181.14 KB/185490, heapSize ~475.31 KB/486720, currentSize=20.13 KB/20610 for b35207b85288b14a567277d6fef4534f in 1255ms, sequenceid=369, compaction requested=true 2024-11-22T18:57:49,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b35207b85288b14a567277d6fef4534f: 2024-11-22T18:57:49,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b35207b85288b14a567277d6fef4534f:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T18:57:49,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:57:49,550 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T18:57:49,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b35207b85288b14a567277d6fef4534f:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T18:57:49,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:57:49,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b35207b85288b14a567277d6fef4534f:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T18:57:49,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:57:49,550 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T18:57:49,551 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40127 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T18:57:49,551 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T18:57:49,551 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1541): b35207b85288b14a567277d6fef4534f/A is initiating minor compaction (all files) 2024-11-22T18:57:49,551 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1541): b35207b85288b14a567277d6fef4534f/B is initiating minor compaction (all files) 2024-11-22T18:57:49,551 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b35207b85288b14a567277d6fef4534f/A in TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:49,551 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b35207b85288b14a567277d6fef4534f/B in TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:49,551 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/5abe02d0bb8042bfb74837edc9376b2a, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/10e56784fb4a4ca39cb53a0a8ddf3888, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/91fc73a0da4f46fea5c096f96d61f149] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp, totalSize=39.2 K 2024-11-22T18:57:49,551 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/b4875a75f0dd4deaa620ffd561beaefe, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/ddef69eb82e34182ab58deab34e41a4d, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/b9110ddb2e5e42478395a478ff2c65a9] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp, totalSize=36.8 K 2024-11-22T18:57:49,551 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5abe02d0bb8042bfb74837edc9376b2a, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1732301866055 2024-11-22T18:57:49,551 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting b4875a75f0dd4deaa620ffd561beaefe, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1732301866055 2024-11-22T18:57:49,552 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10e56784fb4a4ca39cb53a0a8ddf3888, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1732301866413 2024-11-22T18:57:49,552 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting ddef69eb82e34182ab58deab34e41a4d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1732301866413 2024-11-22T18:57:49,552 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting b9110ddb2e5e42478395a478ff2c65a9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1732301867643 2024-11-22T18:57:49,552 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91fc73a0da4f46fea5c096f96d61f149, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1732301867617 2024-11-22T18:57:49,559 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b35207b85288b14a567277d6fef4534f#B#compaction#408 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:57:49,559 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/B/04c231097b554b3f89f205d5a33f02da is 50, key is test_row_0/B:col10/1732301867643/Put/seqid=0 2024-11-22T18:57:49,559 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b35207b85288b14a567277d6fef4534f#A#compaction#409 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:57:49,560 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/A/d3cb216bd65d4b2ca2fa048099b7d08d is 50, key is test_row_0/A:col10/1732301867643/Put/seqid=0 2024-11-22T18:57:49,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742296_1472 (size=13187) 2024-11-22T18:57:49,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742297_1473 (size=13187) 2024-11-22T18:57:49,572 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/A/d3cb216bd65d4b2ca2fa048099b7d08d as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/d3cb216bd65d4b2ca2fa048099b7d08d 2024-11-22T18:57:49,576 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b35207b85288b14a567277d6fef4534f/A of b35207b85288b14a567277d6fef4534f into d3cb216bd65d4b2ca2fa048099b7d08d(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:57:49,576 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b35207b85288b14a567277d6fef4534f: 2024-11-22T18:57:49,576 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., storeName=b35207b85288b14a567277d6fef4534f/A, priority=13, startTime=1732301869549; duration=0sec 2024-11-22T18:57:49,576 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:57:49,576 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b35207b85288b14a567277d6fef4534f:A 2024-11-22T18:57:49,577 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T18:57:49,577 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T18:57:49,577 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1541): b35207b85288b14a567277d6fef4534f/C is initiating minor compaction (all files) 2024-11-22T18:57:49,577 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b35207b85288b14a567277d6fef4534f/C in TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:49,577 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/87d18eea275942c7ae14c2edbb11974a, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/3e8d196976f14b6fa938ff1619ce74ae, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/813438c61e3841b782cea760d0ed120d] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp, totalSize=36.8 K 2024-11-22T18:57:49,578 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87d18eea275942c7ae14c2edbb11974a, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1732301866055 2024-11-22T18:57:49,578 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e8d196976f14b6fa938ff1619ce74ae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1732301866413 2024-11-22T18:57:49,578 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 813438c61e3841b782cea760d0ed120d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1732301867643 2024-11-22T18:57:49,590 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b35207b85288b14a567277d6fef4534f#C#compaction#410 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:57:49,591 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/C/45d643c473c2427fb152613b477f8415 is 50, key is test_row_0/C:col10/1732301867643/Put/seqid=0 2024-11-22T18:57:49,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742298_1474 (size=13187) 2024-11-22T18:57:49,607 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=152 2024-11-22T18:57:49,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:49,607 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.HRegion(2902): Flushing b35207b85288b14a567277d6fef4534f 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-22T18:57:49,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/A/33a4f6b0a46d4969a9251f564cc3f7e9 is 50, key is test_row_0/A:col10/1732301868297/Put/seqid=0 2024-11-22T18:57:49,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742299_1475 (size=9857) 2024-11-22T18:57:49,857 INFO [Thread-1791 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1791. 2024-11-22T18:57:49,857 DEBUG [Thread-1791 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T18:57:49,857 DEBUG [Thread-1791 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:57:49,857 DEBUG [Thread-1791 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:57:49,857 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T18:57:49,972 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/B/04c231097b554b3f89f205d5a33f02da as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/04c231097b554b3f89f205d5a33f02da 2024-11-22T18:57:49,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=151 2024-11-22T18:57:49,976 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b35207b85288b14a567277d6fef4534f/B of b35207b85288b14a567277d6fef4534f into 04c231097b554b3f89f205d5a33f02da(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:57:49,976 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b35207b85288b14a567277d6fef4534f: 2024-11-22T18:57:49,976 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., storeName=b35207b85288b14a567277d6fef4534f/B, priority=13, startTime=1732301869550; duration=0sec 2024-11-22T18:57:49,976 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:57:49,976 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b35207b85288b14a567277d6fef4534f:B 2024-11-22T18:57:50,006 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/C/45d643c473c2427fb152613b477f8415 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/45d643c473c2427fb152613b477f8415 2024-11-22T18:57:50,010 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b35207b85288b14a567277d6fef4534f/C of b35207b85288b14a567277d6fef4534f into 45d643c473c2427fb152613b477f8415(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:57:50,010 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b35207b85288b14a567277d6fef4534f: 2024-11-22T18:57:50,010 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., storeName=b35207b85288b14a567277d6fef4534f/C, priority=13, startTime=1732301869550; duration=0sec 2024-11-22T18:57:50,010 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:57:50,011 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b35207b85288b14a567277d6fef4534f:C 2024-11-22T18:57:50,027 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/A/33a4f6b0a46d4969a9251f564cc3f7e9 2024-11-22T18:57:50,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/B/bc405e1572f94745a53bc98640349f9c is 50, key is test_row_0/B:col10/1732301868297/Put/seqid=0 2024-11-22T18:57:50,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742300_1476 (size=9857) 2024-11-22T18:57:50,437 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/B/bc405e1572f94745a53bc98640349f9c 2024-11-22T18:57:50,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/C/57496cfd01fb495b8cfe4473636bc55b is 50, key is test_row_0/C:col10/1732301868297/Put/seqid=0 2024-11-22T18:57:50,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742301_1477 (size=9857) 2024-11-22T18:57:50,478 INFO [Thread-1789 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1789. 2024-11-22T18:57:50,478 DEBUG [Thread-1789 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T18:57:50,478 INFO [Thread-1785 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1785. 2024-11-22T18:57:50,478 DEBUG [Thread-1789 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:57:50,478 DEBUG [Thread-1785 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T18:57:50,478 DEBUG [Thread-1785 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:57:50,478 DEBUG [Thread-1785 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:57:50,478 DEBUG [Thread-1789 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:57:50,478 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T18:57:50,478 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T18:57:50,846 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/C/57496cfd01fb495b8cfe4473636bc55b 2024-11-22T18:57:50,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/A/33a4f6b0a46d4969a9251f564cc3f7e9 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/33a4f6b0a46d4969a9251f564cc3f7e9 2024-11-22T18:57:50,854 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/33a4f6b0a46d4969a9251f564cc3f7e9, entries=100, sequenceid=376, filesize=9.6 K 2024-11-22T18:57:50,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/B/bc405e1572f94745a53bc98640349f9c as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/bc405e1572f94745a53bc98640349f9c 2024-11-22T18:57:50,858 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/bc405e1572f94745a53bc98640349f9c, entries=100, sequenceid=376, filesize=9.6 K 2024-11-22T18:57:50,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/C/57496cfd01fb495b8cfe4473636bc55b as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/57496cfd01fb495b8cfe4473636bc55b 2024-11-22T18:57:50,862 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/57496cfd01fb495b8cfe4473636bc55b, entries=100, sequenceid=376, filesize=9.6 K 2024-11-22T18:57:50,863 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=20.13 KB/20610 for b35207b85288b14a567277d6fef4534f in 1256ms, sequenceid=376, compaction requested=false 2024-11-22T18:57:50,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.HRegion(2603): Flush status journal for b35207b85288b14a567277d6fef4534f: 2024-11-22T18:57:50,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:50,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=152 2024-11-22T18:57:50,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4169): Remote procedure done, pid=152 2024-11-22T18:57:50,866 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=152, resume processing ppid=151 2024-11-22T18:57:50,866 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, ppid=151, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0250 sec 2024-11-22T18:57:50,867 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=151, table=TestAcidGuarantees in 2.0290 sec 2024-11-22T18:57:50,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=151 2024-11-22T18:57:50,987 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T18:57:52,097 INFO [Thread-1783 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1783. 2024-11-22T18:57:52,097 DEBUG [Thread-1783 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T18:57:52,097 DEBUG [Thread-1783 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:57:52,097 DEBUG [Thread-1783 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:57:52,097 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T18:57:52,429 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-22T18:57:55,537 INFO [Thread-1787 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1787. 2024-11-22T18:57:55,537 DEBUG [Thread-1787 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T18:57:55,538 DEBUG [Thread-1787 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:57:55,538 DEBUG [Thread-1787 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:57:55,538 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T18:57:55,538 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-22T18:57:55,538 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 23 2024-11-22T18:57:55,538 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 79 2024-11-22T18:57:55,538 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 24 2024-11-22T18:57:55,538 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 81 2024-11-22T18:57:55,538 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 89 2024-11-22T18:57:55,538 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-22T18:57:55,538 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-22T18:57:55,538 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1394 2024-11-22T18:57:55,538 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4181 rows 2024-11-22T18:57:55,538 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1399 2024-11-22T18:57:55,538 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4191 rows 2024-11-22T18:57:55,538 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1400 2024-11-22T18:57:55,538 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4199 rows 2024-11-22T18:57:55,538 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1394 2024-11-22T18:57:55,538 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4176 rows 2024-11-22T18:57:55,538 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1401 2024-11-22T18:57:55,538 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4202 rows 2024-11-22T18:57:55,538 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-22T18:57:55,538 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T18:57:55,538 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:57:55,538 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:57:55,539 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T18:57:55,539 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-22T18:57:55,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] procedure2.ProcedureExecutor(1139): Stored pid=153, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-22T18:57:55,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=153 2024-11-22T18:57:55,542 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732301875541"}]},"ts":"1732301875541"} 2024-11-22T18:57:55,543 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-22T18:57:55,543 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-22T18:57:55,543 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=154, ppid=153, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-22T18:57:55,545 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b35207b85288b14a567277d6fef4534f, UNASSIGN}] 2024-11-22T18:57:55,545 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=155, ppid=154, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b35207b85288b14a567277d6fef4534f, UNASSIGN 2024-11-22T18:57:55,546 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=155 updating hbase:meta row=b35207b85288b14a567277d6fef4534f, regionState=CLOSING, regionLocation=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:55,547 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=155, ppid=154, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b35207b85288b14a567277d6fef4534f, UNASSIGN because future has completed 2024-11-22T18:57:55,547 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-22T18:57:55,547 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=156, ppid=155, state=RUNNABLE, hasLock=false; CloseRegionProcedure b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173}] 2024-11-22T18:57:55,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=153 2024-11-22T18:57:55,699 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] handler.UnassignRegionHandler(122): Close b35207b85288b14a567277d6fef4534f 2024-11-22T18:57:55,699 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-22T18:57:55,699 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1722): Closing b35207b85288b14a567277d6fef4534f, disabling compactions & flushes 2024-11-22T18:57:55,699 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:55,699 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:55,699 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. after waiting 0 ms 2024-11-22T18:57:55,699 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:55,699 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(2902): Flushing b35207b85288b14a567277d6fef4534f 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-22T18:57:55,702 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/A/cd9bea8e290f4f8ab40a81d505c5efde is 50, key is test_row_0/A:col10/1732301875536/Put/seqid=0 2024-11-22T18:57:55,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742302_1478 (size=9857) 2024-11-22T18:57:55,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=153 2024-11-22T18:57:56,106 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/A/cd9bea8e290f4f8ab40a81d505c5efde 2024-11-22T18:57:56,113 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/B/ad7a2afaff3041648cd245847c1207dd is 50, key is test_row_0/B:col10/1732301875536/Put/seqid=0 2024-11-22T18:57:56,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742303_1479 (size=9857) 2024-11-22T18:57:56,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=153 2024-11-22T18:57:56,516 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/B/ad7a2afaff3041648cd245847c1207dd 2024-11-22T18:57:56,522 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/C/0438a4d6c23d4bbcbe73c277b36c0faf is 50, key is test_row_0/C:col10/1732301875536/Put/seqid=0 2024-11-22T18:57:56,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742304_1480 (size=9857) 2024-11-22T18:57:56,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=153 2024-11-22T18:57:56,926 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/C/0438a4d6c23d4bbcbe73c277b36c0faf 2024-11-22T18:57:56,929 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/A/cd9bea8e290f4f8ab40a81d505c5efde as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/cd9bea8e290f4f8ab40a81d505c5efde 2024-11-22T18:57:56,932 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/cd9bea8e290f4f8ab40a81d505c5efde, entries=100, sequenceid=386, filesize=9.6 K 2024-11-22T18:57:56,933 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/B/ad7a2afaff3041648cd245847c1207dd as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/ad7a2afaff3041648cd245847c1207dd 2024-11-22T18:57:56,935 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/ad7a2afaff3041648cd245847c1207dd, entries=100, sequenceid=386, filesize=9.6 K 2024-11-22T18:57:56,936 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/.tmp/C/0438a4d6c23d4bbcbe73c277b36c0faf as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/0438a4d6c23d4bbcbe73c277b36c0faf 2024-11-22T18:57:56,938 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/0438a4d6c23d4bbcbe73c277b36c0faf, entries=100, sequenceid=386, filesize=9.6 K 2024-11-22T18:57:56,939 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for b35207b85288b14a567277d6fef4534f in 1240ms, sequenceid=386, compaction requested=true 2024-11-22T18:57:56,939 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/da8513b04689423fbb6d9d225f926ca8, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/efbbc5eb3ee446538a289b7f1426a8f8, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/9765b061d9ac4549a6e7fc178f7e75e5, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/452664526c4d4f879acab6cdf284cebe, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/37b010b384a143819c2eaccbd7a9555d, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/2ec8fb05646a4c20b4f1bbbd5e352133, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/98088ebb1f5247a8b6b4334ff635f3ac, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/864d5c4314444664a88f487e6e67ae82, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/e9a06511de1c446ca8ebd9d0b1590344, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/0166be198db14acb88971e0eb795b426, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/177ff348c93d47c0af3c72b43ce3e0e9, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/36d9b8ae99c945c5b206130cf792fe6f, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/6d7d921517c94df7a5070518c4e63110, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/495816b50c274d54b540ded95179125e, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/6ff02883b40549b5a870ad8ddaed5b16, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/bc8c28f1285242df90d34e0b4c72e71e, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/ae0c04f92cac4e37bb7265e6103c77da, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/62a182371b494d00a00f405bab8a0c1e, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/1bad2857105246e98935398e1af0ba7f, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/c1acf6de814b4f23827847c15c122142, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/5adc3072171d4baf9a599de0ffa50f2c, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/95c3f947398549ab99f4beb95eac423e, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/e34eff47556747089ba08f99671c37ee, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/5abe02d0bb8042bfb74837edc9376b2a, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/10e56784fb4a4ca39cb53a0a8ddf3888, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/91fc73a0da4f46fea5c096f96d61f149] to archive 2024-11-22T18:57:56,940 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T18:57:56,941 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/da8513b04689423fbb6d9d225f926ca8 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/da8513b04689423fbb6d9d225f926ca8 2024-11-22T18:57:56,942 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/efbbc5eb3ee446538a289b7f1426a8f8 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/efbbc5eb3ee446538a289b7f1426a8f8 2024-11-22T18:57:56,943 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/9765b061d9ac4549a6e7fc178f7e75e5 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/9765b061d9ac4549a6e7fc178f7e75e5 2024-11-22T18:57:56,943 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/452664526c4d4f879acab6cdf284cebe to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/452664526c4d4f879acab6cdf284cebe 2024-11-22T18:57:56,944 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/37b010b384a143819c2eaccbd7a9555d to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/37b010b384a143819c2eaccbd7a9555d 2024-11-22T18:57:56,945 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/2ec8fb05646a4c20b4f1bbbd5e352133 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/2ec8fb05646a4c20b4f1bbbd5e352133 2024-11-22T18:57:56,946 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/98088ebb1f5247a8b6b4334ff635f3ac to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/98088ebb1f5247a8b6b4334ff635f3ac 2024-11-22T18:57:56,947 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/864d5c4314444664a88f487e6e67ae82 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/864d5c4314444664a88f487e6e67ae82 2024-11-22T18:57:56,947 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/e9a06511de1c446ca8ebd9d0b1590344 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/e9a06511de1c446ca8ebd9d0b1590344 2024-11-22T18:57:56,948 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/0166be198db14acb88971e0eb795b426 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/0166be198db14acb88971e0eb795b426 2024-11-22T18:57:56,949 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/177ff348c93d47c0af3c72b43ce3e0e9 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/177ff348c93d47c0af3c72b43ce3e0e9 2024-11-22T18:57:56,949 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/36d9b8ae99c945c5b206130cf792fe6f to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/36d9b8ae99c945c5b206130cf792fe6f 2024-11-22T18:57:56,950 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/6d7d921517c94df7a5070518c4e63110 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/6d7d921517c94df7a5070518c4e63110 2024-11-22T18:57:56,951 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/495816b50c274d54b540ded95179125e to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/495816b50c274d54b540ded95179125e 2024-11-22T18:57:56,951 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/6ff02883b40549b5a870ad8ddaed5b16 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/6ff02883b40549b5a870ad8ddaed5b16 2024-11-22T18:57:56,952 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/bc8c28f1285242df90d34e0b4c72e71e to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/bc8c28f1285242df90d34e0b4c72e71e 2024-11-22T18:57:56,953 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/ae0c04f92cac4e37bb7265e6103c77da to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/ae0c04f92cac4e37bb7265e6103c77da 2024-11-22T18:57:56,954 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/62a182371b494d00a00f405bab8a0c1e to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/62a182371b494d00a00f405bab8a0c1e 2024-11-22T18:57:56,955 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/1bad2857105246e98935398e1af0ba7f to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/1bad2857105246e98935398e1af0ba7f 2024-11-22T18:57:56,955 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/c1acf6de814b4f23827847c15c122142 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/c1acf6de814b4f23827847c15c122142 2024-11-22T18:57:56,956 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/5adc3072171d4baf9a599de0ffa50f2c to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/5adc3072171d4baf9a599de0ffa50f2c 2024-11-22T18:57:56,957 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/95c3f947398549ab99f4beb95eac423e to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/95c3f947398549ab99f4beb95eac423e 2024-11-22T18:57:56,958 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/e34eff47556747089ba08f99671c37ee to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/e34eff47556747089ba08f99671c37ee 2024-11-22T18:57:56,958 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/5abe02d0bb8042bfb74837edc9376b2a to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/5abe02d0bb8042bfb74837edc9376b2a 2024-11-22T18:57:56,959 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/10e56784fb4a4ca39cb53a0a8ddf3888 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/10e56784fb4a4ca39cb53a0a8ddf3888 2024-11-22T18:57:56,960 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/91fc73a0da4f46fea5c096f96d61f149 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/91fc73a0da4f46fea5c096f96d61f149 2024-11-22T18:57:56,961 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/c9f7459067154f2390049b737bccf6ed, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/ae8ea60c172e41ef9432695dfbfe4aa6, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/ea28d406b46a4687a2c3f2700d35ba7a, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/5f3b80c90edb4a728b40fd4aec96d85d, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/c9f0bc3608c541f59524119a35696b7c, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/9af436e10aa84168ab370ac61ef7d40c, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/55f877c7359b46b08af8089d763c2a87, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/3428fa5b5e3343099e8b7b88e8051792, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/a16133ab1d7a48928d55bbaeb5fcd030, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/325de31a960245e8a960585f2d81f752, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/cd73d11138e746efabd424667d56bdaf, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/5362013ac71f4d5cad07586e3e6ae640, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/e39c40b8e4684fbca85fb263ec7d5f96, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/11deadd861404b208dbdde8e4b4aa851, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/924563c13d0549c4a6015887cac32305, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/319085bf5abb4c38912ca9f0fb08783d, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/f1681babb49648b5ade66ce00dae8f74, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/2507c6661f9e4b3281104ed1b9d92d37, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/e8fc86d75ba2431784d1e169e031076c, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/8661be6db4f24ed48ad7372deb4cb05a, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/92f5cca2383949128d406c045d892c40, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/07a8d0e21f944201b182fa68eea42152, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/b4875a75f0dd4deaa620ffd561beaefe, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/02f3d7a622314f37928b500c7507a6ec, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/ddef69eb82e34182ab58deab34e41a4d, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/b9110ddb2e5e42478395a478ff2c65a9] to archive 2024-11-22T18:57:56,962 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T18:57:56,963 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/c9f7459067154f2390049b737bccf6ed to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/c9f7459067154f2390049b737bccf6ed 2024-11-22T18:57:56,964 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/ae8ea60c172e41ef9432695dfbfe4aa6 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/ae8ea60c172e41ef9432695dfbfe4aa6 2024-11-22T18:57:56,964 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/ea28d406b46a4687a2c3f2700d35ba7a to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/ea28d406b46a4687a2c3f2700d35ba7a 2024-11-22T18:57:56,965 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/5f3b80c90edb4a728b40fd4aec96d85d to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/5f3b80c90edb4a728b40fd4aec96d85d 2024-11-22T18:57:56,966 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/c9f0bc3608c541f59524119a35696b7c to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/c9f0bc3608c541f59524119a35696b7c 2024-11-22T18:57:56,966 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/9af436e10aa84168ab370ac61ef7d40c to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/9af436e10aa84168ab370ac61ef7d40c 2024-11-22T18:57:56,967 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/55f877c7359b46b08af8089d763c2a87 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/55f877c7359b46b08af8089d763c2a87 2024-11-22T18:57:56,968 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/3428fa5b5e3343099e8b7b88e8051792 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/3428fa5b5e3343099e8b7b88e8051792 2024-11-22T18:57:56,969 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/a16133ab1d7a48928d55bbaeb5fcd030 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/a16133ab1d7a48928d55bbaeb5fcd030 2024-11-22T18:57:56,969 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/325de31a960245e8a960585f2d81f752 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/325de31a960245e8a960585f2d81f752 2024-11-22T18:57:56,970 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/cd73d11138e746efabd424667d56bdaf to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/cd73d11138e746efabd424667d56bdaf 2024-11-22T18:57:56,971 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/5362013ac71f4d5cad07586e3e6ae640 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/5362013ac71f4d5cad07586e3e6ae640 2024-11-22T18:57:56,971 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/e39c40b8e4684fbca85fb263ec7d5f96 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/e39c40b8e4684fbca85fb263ec7d5f96 2024-11-22T18:57:56,972 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/11deadd861404b208dbdde8e4b4aa851 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/11deadd861404b208dbdde8e4b4aa851 2024-11-22T18:57:56,973 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/924563c13d0549c4a6015887cac32305 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/924563c13d0549c4a6015887cac32305 2024-11-22T18:57:56,974 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/319085bf5abb4c38912ca9f0fb08783d to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/319085bf5abb4c38912ca9f0fb08783d 2024-11-22T18:57:56,974 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/f1681babb49648b5ade66ce00dae8f74 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/f1681babb49648b5ade66ce00dae8f74 2024-11-22T18:57:56,975 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/2507c6661f9e4b3281104ed1b9d92d37 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/2507c6661f9e4b3281104ed1b9d92d37 2024-11-22T18:57:56,976 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/e8fc86d75ba2431784d1e169e031076c to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/e8fc86d75ba2431784d1e169e031076c 2024-11-22T18:57:56,976 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/8661be6db4f24ed48ad7372deb4cb05a to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/8661be6db4f24ed48ad7372deb4cb05a 2024-11-22T18:57:56,977 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/92f5cca2383949128d406c045d892c40 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/92f5cca2383949128d406c045d892c40 2024-11-22T18:57:56,978 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/07a8d0e21f944201b182fa68eea42152 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/07a8d0e21f944201b182fa68eea42152 2024-11-22T18:57:56,979 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/b4875a75f0dd4deaa620ffd561beaefe to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/b4875a75f0dd4deaa620ffd561beaefe 2024-11-22T18:57:56,979 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/02f3d7a622314f37928b500c7507a6ec to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/02f3d7a622314f37928b500c7507a6ec 2024-11-22T18:57:56,980 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/ddef69eb82e34182ab58deab34e41a4d to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/ddef69eb82e34182ab58deab34e41a4d 2024-11-22T18:57:56,981 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/b9110ddb2e5e42478395a478ff2c65a9 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/b9110ddb2e5e42478395a478ff2c65a9 2024-11-22T18:57:56,982 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/624dcfa3c7a44cf495613edcd1359549, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/d0df4b25458b41f1abf8bb1d7c1c7667, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/30f11592c3524914bb9dc2664a33cc99, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/8b5843c7b78841f79eb8a737ba7941d8, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/afb2eec8ef8e4da498547a0dd1923a4c, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/6dcada98e12f4bdcab4034a2852dcdcc, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/d82dd5892aa94e91ba025a81eb6e2bcf, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/a7f23032ef694650975ecab2352cd341, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/2506525daf5844c498b51392b7f15bb4, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/cf6c3a6fd0214c53aa6d40627b04cd6b, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/76d82c8937ce42ebb82235d1e1c62ac4, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/74c0e30478f64fe3a5b0bac7d175fca1, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/18aa90f3e39947b58cc653b3acbae2df, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/e4c19b0530c94d569aaa16413d06574b, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/418db2535f394a099c0b45e149f2f446, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/ac5093402fad45819f38fc57dc366b17, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/4dcad8574a9940c6b8f54b61a1f95cc6, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/4fbcdd0bf08e452297c6193bbd168d7c, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/c5599aaeca7347e4ae3fa171bc3d849c, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/e8e5840b2ef6470b887485de3693bf30, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/0d2849da796943f59c1d2fa148df9aa4, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/1268a636e30b4b62b7639d24c2237df8, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/87d18eea275942c7ae14c2edbb11974a, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/254b16d6d9d441028a7e28285de4893f, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/3e8d196976f14b6fa938ff1619ce74ae, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/813438c61e3841b782cea760d0ed120d] to archive 2024-11-22T18:57:56,983 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T18:57:56,984 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/624dcfa3c7a44cf495613edcd1359549 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/624dcfa3c7a44cf495613edcd1359549 2024-11-22T18:57:56,985 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/d0df4b25458b41f1abf8bb1d7c1c7667 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/d0df4b25458b41f1abf8bb1d7c1c7667 2024-11-22T18:57:56,985 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/30f11592c3524914bb9dc2664a33cc99 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/30f11592c3524914bb9dc2664a33cc99 2024-11-22T18:57:56,986 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/8b5843c7b78841f79eb8a737ba7941d8 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/8b5843c7b78841f79eb8a737ba7941d8 2024-11-22T18:57:56,987 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/afb2eec8ef8e4da498547a0dd1923a4c to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/afb2eec8ef8e4da498547a0dd1923a4c 2024-11-22T18:57:56,987 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/6dcada98e12f4bdcab4034a2852dcdcc to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/6dcada98e12f4bdcab4034a2852dcdcc 2024-11-22T18:57:56,988 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/d82dd5892aa94e91ba025a81eb6e2bcf to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/d82dd5892aa94e91ba025a81eb6e2bcf 2024-11-22T18:57:56,989 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/a7f23032ef694650975ecab2352cd341 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/a7f23032ef694650975ecab2352cd341 2024-11-22T18:57:56,990 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/2506525daf5844c498b51392b7f15bb4 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/2506525daf5844c498b51392b7f15bb4 2024-11-22T18:57:56,991 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/cf6c3a6fd0214c53aa6d40627b04cd6b to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/cf6c3a6fd0214c53aa6d40627b04cd6b 2024-11-22T18:57:56,991 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/76d82c8937ce42ebb82235d1e1c62ac4 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/76d82c8937ce42ebb82235d1e1c62ac4 2024-11-22T18:57:56,992 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/74c0e30478f64fe3a5b0bac7d175fca1 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/74c0e30478f64fe3a5b0bac7d175fca1 2024-11-22T18:57:56,993 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/18aa90f3e39947b58cc653b3acbae2df to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/18aa90f3e39947b58cc653b3acbae2df 2024-11-22T18:57:56,994 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/e4c19b0530c94d569aaa16413d06574b to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/e4c19b0530c94d569aaa16413d06574b 2024-11-22T18:57:56,994 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/418db2535f394a099c0b45e149f2f446 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/418db2535f394a099c0b45e149f2f446 2024-11-22T18:57:56,995 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/ac5093402fad45819f38fc57dc366b17 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/ac5093402fad45819f38fc57dc366b17 2024-11-22T18:57:56,996 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/4dcad8574a9940c6b8f54b61a1f95cc6 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/4dcad8574a9940c6b8f54b61a1f95cc6 2024-11-22T18:57:56,997 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/4fbcdd0bf08e452297c6193bbd168d7c to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/4fbcdd0bf08e452297c6193bbd168d7c 2024-11-22T18:57:56,998 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/c5599aaeca7347e4ae3fa171bc3d849c to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/c5599aaeca7347e4ae3fa171bc3d849c 2024-11-22T18:57:56,998 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/e8e5840b2ef6470b887485de3693bf30 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/e8e5840b2ef6470b887485de3693bf30 2024-11-22T18:57:56,999 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/0d2849da796943f59c1d2fa148df9aa4 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/0d2849da796943f59c1d2fa148df9aa4 2024-11-22T18:57:57,000 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/1268a636e30b4b62b7639d24c2237df8 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/1268a636e30b4b62b7639d24c2237df8 2024-11-22T18:57:57,001 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/87d18eea275942c7ae14c2edbb11974a to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/87d18eea275942c7ae14c2edbb11974a 2024-11-22T18:57:57,002 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/254b16d6d9d441028a7e28285de4893f to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/254b16d6d9d441028a7e28285de4893f 2024-11-22T18:57:57,003 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/3e8d196976f14b6fa938ff1619ce74ae to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/3e8d196976f14b6fa938ff1619ce74ae 2024-11-22T18:57:57,003 DEBUG [StoreCloser-TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/813438c61e3841b782cea760d0ed120d to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/813438c61e3841b782cea760d0ed120d 2024-11-22T18:57:57,007 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/recovered.edits/389.seqid, newMaxSeqId=389, maxSeqId=1 2024-11-22T18:57:57,008 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. 2024-11-22T18:57:57,008 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1676): Region close journal for b35207b85288b14a567277d6fef4534f: Waiting for close lock at 1732301875699Running coprocessor pre-close hooks at 1732301875699Disabling compacts and flushes for region at 1732301875699Disabling writes for close at 1732301875699Obtaining lock to block concurrent updates at 1732301875699Preparing flush snapshotting stores in b35207b85288b14a567277d6fef4534f at 1732301875699Finished memstore snapshotting TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1732301875699Flushing stores of TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f. at 1732301875700 (+1 ms)Flushing b35207b85288b14a567277d6fef4534f/A: creating writer at 1732301875700Flushing b35207b85288b14a567277d6fef4534f/A: appending metadata at 1732301875702 (+2 ms)Flushing b35207b85288b14a567277d6fef4534f/A: closing flushed file at 1732301875702Flushing b35207b85288b14a567277d6fef4534f/B: creating writer at 1732301876110 (+408 ms)Flushing b35207b85288b14a567277d6fef4534f/B: appending metadata at 1732301876112 (+2 ms)Flushing b35207b85288b14a567277d6fef4534f/B: closing flushed file at 1732301876112Flushing b35207b85288b14a567277d6fef4534f/C: creating writer at 1732301876520 (+408 ms)Flushing b35207b85288b14a567277d6fef4534f/C: appending metadata at 1732301876522 (+2 ms)Flushing b35207b85288b14a567277d6fef4534f/C: closing flushed file at 1732301876522Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1491c8d7: reopening flushed file at 1732301876929 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@13f2d884: reopening flushed file at 1732301876932 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@19dd102c: reopening flushed file at 1732301876936 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for b35207b85288b14a567277d6fef4534f in 1240ms, sequenceid=386, compaction requested=true at 1732301876939 (+3 ms)Writing region close event to WAL at 1732301877005 (+66 ms)Running coprocessor post-close hooks at 1732301877008 (+3 ms)Closed at 1732301877008 2024-11-22T18:57:57,009 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] handler.UnassignRegionHandler(157): Closed b35207b85288b14a567277d6fef4534f 2024-11-22T18:57:57,010 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=155 updating hbase:meta row=b35207b85288b14a567277d6fef4534f, regionState=CLOSED 2024-11-22T18:57:57,011 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=156, ppid=155, state=RUNNABLE, hasLock=false; CloseRegionProcedure b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 because future has completed 2024-11-22T18:57:57,013 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=156, resume processing ppid=155 2024-11-22T18:57:57,013 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, ppid=155, state=SUCCESS, hasLock=false; CloseRegionProcedure b35207b85288b14a567277d6fef4534f, server=d79ba0c344fb,43593,1732301724173 in 1.4650 sec 2024-11-22T18:57:57,014 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-22T18:57:57,014 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b35207b85288b14a567277d6fef4534f, UNASSIGN in 1.4690 sec 2024-11-22T18:57:57,016 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=154, resume processing ppid=153 2024-11-22T18:57:57,016 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, ppid=153, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4720 sec 2024-11-22T18:57:57,017 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732301877017"}]},"ts":"1732301877017"} 2024-11-22T18:57:57,019 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-22T18:57:57,019 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-22T18:57:57,020 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4800 sec 2024-11-22T18:57:57,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=153 2024-11-22T18:57:57,686 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-22T18:57:57,686 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-22T18:57:57,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] procedure2.ProcedureExecutor(1139): Stored pid=157, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T18:57:57,688 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=157, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T18:57:57,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=157 2024-11-22T18:57:57,689 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=157, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T18:57:57,691 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f 2024-11-22T18:57:57,693 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A, FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B, FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C, FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/recovered.edits] 2024-11-22T18:57:57,696 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/33a4f6b0a46d4969a9251f564cc3f7e9 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/33a4f6b0a46d4969a9251f564cc3f7e9 2024-11-22T18:57:57,697 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/cd9bea8e290f4f8ab40a81d505c5efde to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/cd9bea8e290f4f8ab40a81d505c5efde 2024-11-22T18:57:57,698 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/d3cb216bd65d4b2ca2fa048099b7d08d to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/A/d3cb216bd65d4b2ca2fa048099b7d08d 2024-11-22T18:57:57,700 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/04c231097b554b3f89f205d5a33f02da to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/04c231097b554b3f89f205d5a33f02da 2024-11-22T18:57:57,701 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/ad7a2afaff3041648cd245847c1207dd to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/ad7a2afaff3041648cd245847c1207dd 2024-11-22T18:57:57,702 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/bc405e1572f94745a53bc98640349f9c to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/B/bc405e1572f94745a53bc98640349f9c 2024-11-22T18:57:57,705 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/0438a4d6c23d4bbcbe73c277b36c0faf to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/0438a4d6c23d4bbcbe73c277b36c0faf 2024-11-22T18:57:57,706 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/45d643c473c2427fb152613b477f8415 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/45d643c473c2427fb152613b477f8415 2024-11-22T18:57:57,707 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/57496cfd01fb495b8cfe4473636bc55b to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/C/57496cfd01fb495b8cfe4473636bc55b 2024-11-22T18:57:57,709 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/recovered.edits/389.seqid to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f/recovered.edits/389.seqid 2024-11-22T18:57:57,709 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/b35207b85288b14a567277d6fef4534f 2024-11-22T18:57:57,710 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-22T18:57:57,712 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=157, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T18:57:57,713 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-22T18:57:57,715 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-22T18:57:57,716 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=157, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T18:57:57,716 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-22T18:57:57,716 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732301877716"}]},"ts":"9223372036854775807"} 2024-11-22T18:57:57,718 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-22T18:57:57,718 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => b35207b85288b14a567277d6fef4534f, NAME => 'TestAcidGuarantees,,1732301846784.b35207b85288b14a567277d6fef4534f.', STARTKEY => '', ENDKEY => ''}] 2024-11-22T18:57:57,718 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-22T18:57:57,718 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732301877718"}]},"ts":"9223372036854775807"} 2024-11-22T18:57:57,720 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-22T18:57:57,721 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=157, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T18:57:57,721 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 35 msec 2024-11-22T18:57:57,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=157 2024-11-22T18:57:57,796 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-22T18:57:57,796 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-22T18:57:57,805 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=238 (was 240), OpenFileDescriptor=445 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=301 (was 304), ProcessCount=11 (was 11), AvailableMemoryMB=7841 (was 7764) - AvailableMemoryMB LEAK? - 2024-11-22T18:57:57,815 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=238, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=301, ProcessCount=11, AvailableMemoryMB=7840 2024-11-22T18:57:57,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-22T18:57:57,817 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T18:57:57,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-22T18:57:57,819 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=158, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-22T18:57:57,819 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:57:57,819 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 158 2024-11-22T18:57:57,820 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=158, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-22T18:57:57,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-22T18:57:57,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742305_1481 (size=959) 2024-11-22T18:57:57,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-22T18:57:58,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-22T18:57:58,227 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875 2024-11-22T18:57:58,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742306_1482 (size=53) 2024-11-22T18:57:58,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-22T18:57:58,632 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T18:57:58,632 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing cb9c73ed0f14cbdd1f76d53a72aa266b, disabling compactions & flushes 2024-11-22T18:57:58,632 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:57:58,632 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:57:58,632 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. after waiting 0 ms 2024-11-22T18:57:58,632 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:57:58,632 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:57:58,632 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for cb9c73ed0f14cbdd1f76d53a72aa266b: Waiting for close lock at 1732301878632Disabling compacts and flushes for region at 1732301878632Disabling writes for close at 1732301878632Writing region close event to WAL at 1732301878632Closed at 1732301878632 2024-11-22T18:57:58,633 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=158, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-22T18:57:58,633 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732301878633"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732301878633"}]},"ts":"1732301878633"} 2024-11-22T18:57:58,634 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-22T18:57:58,635 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=158, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-22T18:57:58,635 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732301878635"}]},"ts":"1732301878635"} 2024-11-22T18:57:58,636 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-22T18:57:58,636 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cb9c73ed0f14cbdd1f76d53a72aa266b, ASSIGN}] 2024-11-22T18:57:58,637 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=159, ppid=158, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cb9c73ed0f14cbdd1f76d53a72aa266b, ASSIGN 2024-11-22T18:57:58,638 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=159, ppid=158, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cb9c73ed0f14cbdd1f76d53a72aa266b, ASSIGN; state=OFFLINE, location=d79ba0c344fb,43593,1732301724173; forceNewPlan=false, retain=false 2024-11-22T18:57:58,788 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=159 updating hbase:meta row=cb9c73ed0f14cbdd1f76d53a72aa266b, regionState=OPENING, regionLocation=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:58,790 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=159, ppid=158, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cb9c73ed0f14cbdd1f76d53a72aa266b, ASSIGN because future has completed 2024-11-22T18:57:58,790 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=160, ppid=159, state=RUNNABLE, hasLock=false; OpenRegionProcedure cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173}] 2024-11-22T18:57:58,944 INFO [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=160}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:57:58,944 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=160}] regionserver.HRegion(7752): Opening region: {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} 2024-11-22T18:57:58,944 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=160}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees cb9c73ed0f14cbdd1f76d53a72aa266b 2024-11-22T18:57:58,945 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=160}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T18:57:58,945 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=160}] regionserver.HRegion(7794): checking encryption for cb9c73ed0f14cbdd1f76d53a72aa266b 2024-11-22T18:57:58,945 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=160}] regionserver.HRegion(7797): checking classloading for cb9c73ed0f14cbdd1f76d53a72aa266b 2024-11-22T18:57:58,946 INFO [StoreOpener-cb9c73ed0f14cbdd1f76d53a72aa266b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region cb9c73ed0f14cbdd1f76d53a72aa266b 2024-11-22T18:57:58,947 INFO [StoreOpener-cb9c73ed0f14cbdd1f76d53a72aa266b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cb9c73ed0f14cbdd1f76d53a72aa266b columnFamilyName A 2024-11-22T18:57:58,947 DEBUG [StoreOpener-cb9c73ed0f14cbdd1f76d53a72aa266b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:57:58,947 INFO [StoreOpener-cb9c73ed0f14cbdd1f76d53a72aa266b-1 {}] regionserver.HStore(327): Store=cb9c73ed0f14cbdd1f76d53a72aa266b/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T18:57:58,947 INFO [StoreOpener-cb9c73ed0f14cbdd1f76d53a72aa266b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region cb9c73ed0f14cbdd1f76d53a72aa266b 2024-11-22T18:57:58,948 INFO [StoreOpener-cb9c73ed0f14cbdd1f76d53a72aa266b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cb9c73ed0f14cbdd1f76d53a72aa266b columnFamilyName B 2024-11-22T18:57:58,948 DEBUG [StoreOpener-cb9c73ed0f14cbdd1f76d53a72aa266b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:57:58,948 INFO [StoreOpener-cb9c73ed0f14cbdd1f76d53a72aa266b-1 {}] regionserver.HStore(327): Store=cb9c73ed0f14cbdd1f76d53a72aa266b/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T18:57:58,949 INFO [StoreOpener-cb9c73ed0f14cbdd1f76d53a72aa266b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region cb9c73ed0f14cbdd1f76d53a72aa266b 2024-11-22T18:57:58,949 INFO [StoreOpener-cb9c73ed0f14cbdd1f76d53a72aa266b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cb9c73ed0f14cbdd1f76d53a72aa266b columnFamilyName C 2024-11-22T18:57:58,949 DEBUG [StoreOpener-cb9c73ed0f14cbdd1f76d53a72aa266b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:57:58,950 INFO [StoreOpener-cb9c73ed0f14cbdd1f76d53a72aa266b-1 {}] regionserver.HStore(327): Store=cb9c73ed0f14cbdd1f76d53a72aa266b/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T18:57:58,950 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=160}] regionserver.HRegion(1038): replaying wal for cb9c73ed0f14cbdd1f76d53a72aa266b 2024-11-22T18:57:58,950 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=160}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b 2024-11-22T18:57:58,951 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=160}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b 2024-11-22T18:57:58,951 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=160}] regionserver.HRegion(1048): stopping wal replay for cb9c73ed0f14cbdd1f76d53a72aa266b 2024-11-22T18:57:58,951 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=160}] regionserver.HRegion(1060): Cleaning up temporary data for cb9c73ed0f14cbdd1f76d53a72aa266b 2024-11-22T18:57:58,952 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=160}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-22T18:57:58,953 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=160}] regionserver.HRegion(1093): writing seq id for cb9c73ed0f14cbdd1f76d53a72aa266b 2024-11-22T18:57:58,954 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=160}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T18:57:58,954 INFO [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=160}] regionserver.HRegion(1114): Opened cb9c73ed0f14cbdd1f76d53a72aa266b; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59587380, jitterRate=-0.11207884550094604}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-22T18:57:58,954 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=160}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cb9c73ed0f14cbdd1f76d53a72aa266b 2024-11-22T18:57:58,955 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=160}] regionserver.HRegion(1006): Region open journal for cb9c73ed0f14cbdd1f76d53a72aa266b: Running coprocessor pre-open hook at 1732301878945Writing region info on filesystem at 1732301878945Initializing all the Stores at 1732301878945Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732301878945Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732301878945Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732301878945Cleaning up temporary data from old regions at 1732301878951 (+6 ms)Running coprocessor post-open hooks at 1732301878954 (+3 ms)Region opened successfully at 1732301878955 (+1 ms) 2024-11-22T18:57:58,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-22T18:57:58,955 INFO [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=160}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., pid=160, masterSystemTime=1732301878941 2024-11-22T18:57:58,957 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=160}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:57:58,957 INFO [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=160}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:57:58,957 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=159 updating hbase:meta row=cb9c73ed0f14cbdd1f76d53a72aa266b, regionState=OPEN, openSeqNum=2, regionLocation=d79ba0c344fb,43593,1732301724173 2024-11-22T18:57:58,958 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=160, ppid=159, state=RUNNABLE, hasLock=false; OpenRegionProcedure cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 because future has completed 2024-11-22T18:57:58,960 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=160, resume processing ppid=159 2024-11-22T18:57:58,960 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, ppid=159, state=SUCCESS, hasLock=false; OpenRegionProcedure cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 in 169 msec 2024-11-22T18:57:58,961 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-22T18:57:58,961 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cb9c73ed0f14cbdd1f76d53a72aa266b, ASSIGN in 324 msec 2024-11-22T18:57:58,962 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=158, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-22T18:57:58,962 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732301878962"}]},"ts":"1732301878962"} 2024-11-22T18:57:58,963 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-22T18:57:58,964 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=158, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-22T18:57:58,965 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1470 sec 2024-11-22T18:57:59,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-22T18:57:59,976 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-22T18:57:59,976 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1832522d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:57:59,977 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d79ba0c344fb,44109,-1 for getting cluster id 2024-11-22T18:57:59,977 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T18:57:59,978 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '058c0dbe-7061-419a-804d-605a1c1b6341' 2024-11-22T18:57:59,978 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T18:57:59,978 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "058c0dbe-7061-419a-804d-605a1c1b6341" 2024-11-22T18:57:59,978 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45159419, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:57:59,978 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d79ba0c344fb,44109,-1] 2024-11-22T18:57:59,978 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T18:57:59,978 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:57:59,979 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39918, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T18:57:59,980 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@db6838b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:57:59,980 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T18:57:59,981 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d79ba0c344fb,43593,1732301724173, seqNum=-1] 2024-11-22T18:57:59,981 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T18:57:59,981 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32990, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T18:57:59,982 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-22T18:57:59,983 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is d79ba0c344fb,44109,1732301723398 2024-11-22T18:57:59,983 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@153c909e 2024-11-22T18:57:59,983 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-22T18:57:59,983 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39924, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-22T18:57:59,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-22T18:57:59,985 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T18:57:59,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] procedure2.ProcedureExecutor(1139): Stored pid=161, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-22T18:57:59,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742307_1483 (size=995) 2024-11-22T18:58:00,395 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-22T18:58:00,395 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-22T18:58:00,397 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=162, ppid=161, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-22T18:58:00,399 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cb9c73ed0f14cbdd1f76d53a72aa266b, REOPEN/MOVE}] 2024-11-22T18:58:00,399 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=163, ppid=162, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cb9c73ed0f14cbdd1f76d53a72aa266b, REOPEN/MOVE 2024-11-22T18:58:00,400 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=163 updating hbase:meta row=cb9c73ed0f14cbdd1f76d53a72aa266b, regionState=CLOSING, regionLocation=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:00,401 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=163, ppid=162, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cb9c73ed0f14cbdd1f76d53a72aa266b, REOPEN/MOVE because future has completed 2024-11-22T18:58:00,401 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-22T18:58:00,401 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=164, ppid=163, state=RUNNABLE, hasLock=false; CloseRegionProcedure cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173}] 2024-11-22T18:58:00,553 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] handler.UnassignRegionHandler(122): Close cb9c73ed0f14cbdd1f76d53a72aa266b 2024-11-22T18:58:00,553 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-22T18:58:00,553 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] regionserver.HRegion(1722): Closing cb9c73ed0f14cbdd1f76d53a72aa266b, disabling compactions & flushes 2024-11-22T18:58:00,553 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:00,553 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:00,553 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. after waiting 0 ms 2024-11-22T18:58:00,553 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:00,559 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-22T18:58:00,560 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:00,560 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] regionserver.HRegion(1676): Region close journal for cb9c73ed0f14cbdd1f76d53a72aa266b: Waiting for close lock at 1732301880553Running coprocessor pre-close hooks at 1732301880553Disabling compacts and flushes for region at 1732301880553Disabling writes for close at 1732301880553Writing region close event to WAL at 1732301880554 (+1 ms)Running coprocessor post-close hooks at 1732301880560 (+6 ms)Closed at 1732301880560 2024-11-22T18:58:00,560 WARN [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] regionserver.HRegionServer(3299): Not adding moved region record: cb9c73ed0f14cbdd1f76d53a72aa266b to self. 2024-11-22T18:58:00,561 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=164}] handler.UnassignRegionHandler(157): Closed cb9c73ed0f14cbdd1f76d53a72aa266b 2024-11-22T18:58:00,562 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=163 updating hbase:meta row=cb9c73ed0f14cbdd1f76d53a72aa266b, regionState=CLOSED 2024-11-22T18:58:00,563 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=164, ppid=163, state=RUNNABLE, hasLock=false; CloseRegionProcedure cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 because future has completed 2024-11-22T18:58:00,565 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=164, resume processing ppid=163 2024-11-22T18:58:00,565 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, ppid=163, state=SUCCESS, hasLock=false; CloseRegionProcedure cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 in 163 msec 2024-11-22T18:58:00,566 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=163, ppid=162, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cb9c73ed0f14cbdd1f76d53a72aa266b, REOPEN/MOVE; state=CLOSED, location=d79ba0c344fb,43593,1732301724173; forceNewPlan=false, retain=true 2024-11-22T18:58:00,716 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=163 updating hbase:meta row=cb9c73ed0f14cbdd1f76d53a72aa266b, regionState=OPENING, regionLocation=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:00,718 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=163, ppid=162, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cb9c73ed0f14cbdd1f76d53a72aa266b, REOPEN/MOVE because future has completed 2024-11-22T18:58:00,718 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=163, state=RUNNABLE, hasLock=false; OpenRegionProcedure cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173}] 2024-11-22T18:58:00,873 INFO [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=165}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:00,873 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=165}] regionserver.HRegion(7752): Opening region: {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} 2024-11-22T18:58:00,873 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=165}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees cb9c73ed0f14cbdd1f76d53a72aa266b 2024-11-22T18:58:00,873 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=165}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T18:58:00,873 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=165}] regionserver.HRegion(7794): checking encryption for cb9c73ed0f14cbdd1f76d53a72aa266b 2024-11-22T18:58:00,873 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=165}] regionserver.HRegion(7797): checking classloading for cb9c73ed0f14cbdd1f76d53a72aa266b 2024-11-22T18:58:00,875 INFO [StoreOpener-cb9c73ed0f14cbdd1f76d53a72aa266b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region cb9c73ed0f14cbdd1f76d53a72aa266b 2024-11-22T18:58:00,875 INFO [StoreOpener-cb9c73ed0f14cbdd1f76d53a72aa266b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cb9c73ed0f14cbdd1f76d53a72aa266b columnFamilyName A 2024-11-22T18:58:00,876 DEBUG [StoreOpener-cb9c73ed0f14cbdd1f76d53a72aa266b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:00,877 INFO [StoreOpener-cb9c73ed0f14cbdd1f76d53a72aa266b-1 {}] regionserver.HStore(327): Store=cb9c73ed0f14cbdd1f76d53a72aa266b/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T18:58:00,877 INFO [StoreOpener-cb9c73ed0f14cbdd1f76d53a72aa266b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region cb9c73ed0f14cbdd1f76d53a72aa266b 2024-11-22T18:58:00,878 INFO [StoreOpener-cb9c73ed0f14cbdd1f76d53a72aa266b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cb9c73ed0f14cbdd1f76d53a72aa266b columnFamilyName B 2024-11-22T18:58:00,878 DEBUG [StoreOpener-cb9c73ed0f14cbdd1f76d53a72aa266b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:00,878 INFO [StoreOpener-cb9c73ed0f14cbdd1f76d53a72aa266b-1 {}] regionserver.HStore(327): Store=cb9c73ed0f14cbdd1f76d53a72aa266b/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T18:58:00,879 INFO [StoreOpener-cb9c73ed0f14cbdd1f76d53a72aa266b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region cb9c73ed0f14cbdd1f76d53a72aa266b 2024-11-22T18:58:00,879 INFO [StoreOpener-cb9c73ed0f14cbdd1f76d53a72aa266b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cb9c73ed0f14cbdd1f76d53a72aa266b columnFamilyName C 2024-11-22T18:58:00,879 DEBUG [StoreOpener-cb9c73ed0f14cbdd1f76d53a72aa266b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:00,879 INFO [StoreOpener-cb9c73ed0f14cbdd1f76d53a72aa266b-1 {}] regionserver.HStore(327): Store=cb9c73ed0f14cbdd1f76d53a72aa266b/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T18:58:00,880 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=165}] regionserver.HRegion(1038): replaying wal for cb9c73ed0f14cbdd1f76d53a72aa266b 2024-11-22T18:58:00,880 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=165}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b 2024-11-22T18:58:00,881 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=165}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b 2024-11-22T18:58:00,882 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=165}] regionserver.HRegion(1048): stopping wal replay for cb9c73ed0f14cbdd1f76d53a72aa266b 2024-11-22T18:58:00,882 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=165}] regionserver.HRegion(1060): Cleaning up temporary data for cb9c73ed0f14cbdd1f76d53a72aa266b 2024-11-22T18:58:00,882 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=165}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-22T18:58:00,883 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=165}] regionserver.HRegion(1093): writing seq id for cb9c73ed0f14cbdd1f76d53a72aa266b 2024-11-22T18:58:00,884 INFO [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=165}] regionserver.HRegion(1114): Opened cb9c73ed0f14cbdd1f76d53a72aa266b; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72124652, jitterRate=0.07474106550216675}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-22T18:58:00,884 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=165}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cb9c73ed0f14cbdd1f76d53a72aa266b 2024-11-22T18:58:00,884 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=165}] regionserver.HRegion(1006): Region open journal for cb9c73ed0f14cbdd1f76d53a72aa266b: Running coprocessor pre-open hook at 1732301880873Writing region info on filesystem at 1732301880873Initializing all the Stores at 1732301880874 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732301880874Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732301880874Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732301880874Cleaning up temporary data from old regions at 1732301880882 (+8 ms)Running coprocessor post-open hooks at 1732301880884 (+2 ms)Region opened successfully at 1732301880884 2024-11-22T18:58:00,885 INFO [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=165}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., pid=165, masterSystemTime=1732301880870 2024-11-22T18:58:00,887 DEBUG [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=165}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:00,887 INFO [RS_OPEN_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_OPEN_REGION, pid=165}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:00,887 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=163 updating hbase:meta row=cb9c73ed0f14cbdd1f76d53a72aa266b, regionState=OPEN, openSeqNum=5, regionLocation=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:00,889 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=165, ppid=163, state=RUNNABLE, hasLock=false; OpenRegionProcedure cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 because future has completed 2024-11-22T18:58:00,890 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=163 2024-11-22T18:58:00,890 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=163, state=SUCCESS, hasLock=false; OpenRegionProcedure cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 in 171 msec 2024-11-22T18:58:00,892 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-22T18:58:00,892 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cb9c73ed0f14cbdd1f76d53a72aa266b, REOPEN/MOVE in 491 msec 2024-11-22T18:58:00,893 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=162, resume processing ppid=161 2024-11-22T18:58:00,894 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, ppid=161, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 495 msec 2024-11-22T18:58:00,895 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 909 msec 2024-11-22T18:58:00,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=161 2024-11-22T18:58:00,896 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-22T18:58:00,897 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22faf5ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:58:00,897 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d79ba0c344fb,44109,-1 for getting cluster id 2024-11-22T18:58:00,897 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T18:58:00,897 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '058c0dbe-7061-419a-804d-605a1c1b6341' 2024-11-22T18:58:00,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T18:58:00,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "058c0dbe-7061-419a-804d-605a1c1b6341" 2024-11-22T18:58:00,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@710a52e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:58:00,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d79ba0c344fb,44109,-1] 2024-11-22T18:58:00,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T18:58:00,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:58:00,899 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32856, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T18:58:00,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58cb7d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:58:00,900 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@744b0f71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:58:00,900 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d79ba0c344fb,44109,-1 for getting cluster id 2024-11-22T18:58:00,900 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T18:58:00,900 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '058c0dbe-7061-419a-804d-605a1c1b6341' 2024-11-22T18:58:00,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T18:58:00,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "058c0dbe-7061-419a-804d-605a1c1b6341" 2024-11-22T18:58:00,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1586ab0c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:58:00,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d79ba0c344fb,44109,-1] 2024-11-22T18:58:00,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T18:58:00,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:58:00,901 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32886, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T18:58:00,902 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74d2cf1d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:58:00,902 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@143fb08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:58:00,902 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d79ba0c344fb,44109,-1 for getting cluster id 2024-11-22T18:58:00,902 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T18:58:00,903 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '058c0dbe-7061-419a-804d-605a1c1b6341' 2024-11-22T18:58:00,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T18:58:00,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "058c0dbe-7061-419a-804d-605a1c1b6341" 2024-11-22T18:58:00,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4635500d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:58:00,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d79ba0c344fb,44109,-1] 2024-11-22T18:58:00,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T18:58:00,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:58:00,904 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32906, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T18:58:00,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a7da706, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:58:00,905 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40503a36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:58:00,905 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d79ba0c344fb,44109,-1 for getting cluster id 2024-11-22T18:58:00,905 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T18:58:00,905 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '058c0dbe-7061-419a-804d-605a1c1b6341' 2024-11-22T18:58:00,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T18:58:00,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "058c0dbe-7061-419a-804d-605a1c1b6341" 2024-11-22T18:58:00,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64b50d43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:58:00,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d79ba0c344fb,44109,-1] 2024-11-22T18:58:00,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T18:58:00,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:58:00,906 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32936, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T18:58:00,907 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78c3efc7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:58:00,907 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52f72e1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:58:00,907 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d79ba0c344fb,44109,-1 for getting cluster id 2024-11-22T18:58:00,907 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T18:58:00,908 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '058c0dbe-7061-419a-804d-605a1c1b6341' 2024-11-22T18:58:00,908 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T18:58:00,908 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "058c0dbe-7061-419a-804d-605a1c1b6341" 2024-11-22T18:58:00,908 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5173f9c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:58:00,908 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d79ba0c344fb,44109,-1] 2024-11-22T18:58:00,908 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T18:58:00,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:58:00,909 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32952, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T18:58:00,909 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@577e8c2f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:58:00,910 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@763ca8cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:58:00,910 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d79ba0c344fb,44109,-1 for getting cluster id 2024-11-22T18:58:00,910 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T18:58:00,911 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '058c0dbe-7061-419a-804d-605a1c1b6341' 2024-11-22T18:58:00,911 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T18:58:00,911 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "058c0dbe-7061-419a-804d-605a1c1b6341" 2024-11-22T18:58:00,911 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3879846c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:58:00,911 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d79ba0c344fb,44109,-1] 2024-11-22T18:58:00,911 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T18:58:00,911 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:58:00,912 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32972, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T18:58:00,912 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@607a4f00, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:58:00,913 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7122db2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:58:00,913 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d79ba0c344fb,44109,-1 for getting cluster id 2024-11-22T18:58:00,913 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T18:58:00,913 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '058c0dbe-7061-419a-804d-605a1c1b6341' 2024-11-22T18:58:00,913 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T18:58:00,913 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "058c0dbe-7061-419a-804d-605a1c1b6341" 2024-11-22T18:58:00,914 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37f3d431, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:58:00,914 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d79ba0c344fb,44109,-1] 2024-11-22T18:58:00,914 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T18:58:00,914 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:58:00,914 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33000, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T18:58:00,915 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f4d50ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:58:00,915 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3666b173, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:58:00,915 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d79ba0c344fb,44109,-1 for getting cluster id 2024-11-22T18:58:00,915 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T18:58:00,916 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '058c0dbe-7061-419a-804d-605a1c1b6341' 2024-11-22T18:58:00,916 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T18:58:00,916 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "058c0dbe-7061-419a-804d-605a1c1b6341" 2024-11-22T18:58:00,916 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d132507, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:58:00,916 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d79ba0c344fb,44109,-1] 2024-11-22T18:58:00,916 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T18:58:00,916 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:58:00,917 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33028, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T18:58:00,917 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@124de015, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:58:00,918 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77ff027c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:58:00,918 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d79ba0c344fb,44109,-1 for getting cluster id 2024-11-22T18:58:00,918 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T18:58:00,918 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '058c0dbe-7061-419a-804d-605a1c1b6341' 2024-11-22T18:58:00,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T18:58:00,919 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "058c0dbe-7061-419a-804d-605a1c1b6341" 2024-11-22T18:58:00,919 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5da5ec40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:58:00,919 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d79ba0c344fb,44109,-1] 2024-11-22T18:58:00,919 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T18:58:00,919 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:58:00,920 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33044, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T18:58:00,920 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39d58e55, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:58:00,920 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2637386d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:58:00,920 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d79ba0c344fb,44109,-1 for getting cluster id 2024-11-22T18:58:00,920 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T18:58:00,921 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '058c0dbe-7061-419a-804d-605a1c1b6341' 2024-11-22T18:58:00,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T18:58:00,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "058c0dbe-7061-419a-804d-605a1c1b6341" 2024-11-22T18:58:00,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f8c7ef0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:58:00,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d79ba0c344fb,44109,-1] 2024-11-22T18:58:00,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T18:58:00,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:58:00,922 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33070, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T18:58:00,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52e009da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T18:58:00,923 DEBUG [Thread-2191 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T18:58:00,923 DEBUG [Thread-2174 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T18:58:00,923 DEBUG [Thread-2189 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T18:58:00,923 DEBUG [Thread-2178 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T18:58:00,924 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T18:58:00,924 DEBUG [Thread-2183 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T18:58:00,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d79ba0c344fb,43593,1732301724173, seqNum=-1] 2024-11-22T18:58:00,924 DEBUG [Thread-2180 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T18:58:00,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T18:58:00,924 DEBUG [Thread-2185 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T18:58:00,924 DEBUG [Thread-2189 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d79ba0c344fb,43593,1732301724173, seqNum=-1] 2024-11-22T18:58:00,925 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d79ba0c344fb,43593,1732301724173, seqNum=-1] 2024-11-22T18:58:00,925 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d79ba0c344fb,43593,1732301724173, seqNum=-1] 2024-11-22T18:58:00,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-11-22T18:58:00,925 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T18:58:00,925 DEBUG [Thread-2172 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T18:58:00,925 DEBUG [Thread-2187 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T18:58:00,925 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T18:58:00,925 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34518, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T18:58:00,925 DEBUG [Thread-2176 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T18:58:00,925 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d79ba0c344fb,43593,1732301724173, seqNum=-1] 2024-11-22T18:58:00,926 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T18:58:00,926 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34524, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T18:58:00,926 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d79ba0c344fb,43593,1732301724173, seqNum=-1] 2024-11-22T18:58:00,926 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d79ba0c344fb,43593,1732301724173, seqNum=-1] 2024-11-22T18:58:00,926 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d79ba0c344fb,43593,1732301724173, seqNum=-1] 2024-11-22T18:58:00,926 DEBUG [Thread-2189 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T18:58:00,926 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34522, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T18:58:00,926 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T18:58:00,926 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T18:58:00,926 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T18:58:00,927 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T18:58:00,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-22T18:58:00,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5] 2024-11-22T18:58:00,927 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34542, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T18:58:00,927 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34546, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T18:58:00,928 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T18:58:00,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5] 2024-11-22T18:58:00,928 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T18:58:00,928 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34536, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T18:58:00,928 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34548, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T18:58:00,928 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d79ba0c344fb,43593,1732301724173, seqNum=-1] 2024-11-22T18:58:00,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5] 2024-11-22T18:58:00,928 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T18:58:00,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d79ba0c344fb,43593,1732301724173, seqNum=-1] 2024-11-22T18:58:00,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T18:58:00,929 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34564, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T18:58:00,929 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5] 2024-11-22T18:58:00,930 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34568, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T18:58:00,930 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34578, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T18:58:00,930 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5] 2024-11-22T18:58:00,931 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5] 2024-11-22T18:58:00,931 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5] 2024-11-22T18:58:00,932 DEBUG [Thread-2187 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5] 2024-11-22T18:58:00,933 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5] 2024-11-22T18:58:00,933 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5] 2024-11-22T18:58:00,967 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T18:58:00,967 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cb9c73ed0f14cbdd1f76d53a72aa266b 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-22T18:58:00,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on cb9c73ed0f14cbdd1f76d53a72aa266b 2024-11-22T18:58:00,996 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112273fd5e1691f84bd6993550be7cd5225b_cb9c73ed0f14cbdd1f76d53a72aa266b is 50, key is test_row_0/A:col10/1732301880965/Put/seqid=0 2024-11-22T18:58:01,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:01,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34564 deadline: 1732301940994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:01,001 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:01,001 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:01,001 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:01,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:01,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34578 deadline: 1732301940998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:01,002 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:01,002 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:01,002 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:01,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:01,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742308_1484 (size=14594) 2024-11-22T18:58:01,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34568 deadline: 1732301941000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:01,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:01,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:01,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1732301941000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:01,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34524 deadline: 1732301941000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:01,006 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:01,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:01,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:01,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:01,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:01,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:01,007 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:01,007 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:01,007 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:01,007 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:01,012 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112273fd5e1691f84bd6993550be7cd5225b_cb9c73ed0f14cbdd1f76d53a72aa266b to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112273fd5e1691f84bd6993550be7cd5225b_cb9c73ed0f14cbdd1f76d53a72aa266b 2024-11-22T18:58:01,013 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/A/e679fead15da4ede900c00a6dcbd7692, store: [table=TestAcidGuarantees family=A region=cb9c73ed0f14cbdd1f76d53a72aa266b] 2024-11-22T18:58:01,013 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/A/e679fead15da4ede900c00a6dcbd7692 is 175, key is test_row_0/A:col10/1732301880965/Put/seqid=0 2024-11-22T18:58:01,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742309_1485 (size=39549) 2024-11-22T18:58:01,027 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/A/e679fead15da4ede900c00a6dcbd7692 2024-11-22T18:58:01,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-22T18:58:01,078 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/B/f0902f4602d64af09204bbd2487010a1 is 50, key is test_row_0/B:col10/1732301880965/Put/seqid=0 2024-11-22T18:58:01,082 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-22T18:58:01,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:01,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. as already flushing 2024-11-22T18:58:01,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:01,082 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:01,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:01,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:01,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742310_1486 (size=12001) 2024-11-22T18:58:01,099 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/B/f0902f4602d64af09204bbd2487010a1 2024-11-22T18:58:01,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:01,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34564 deadline: 1732301941110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:01,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:01,112 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:01,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34578 deadline: 1732301941110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:01,112 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:01,112 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:01,112 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:01,112 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:01,112 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:01,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:01,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1732301941115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:01,116 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:01,116 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:01,116 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:01,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:01,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34524 deadline: 1732301941115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:01,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:01,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:01,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:01,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:01,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34568 deadline: 1732301941116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:01,118 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:01,118 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:01,118 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:01,129 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/C/02b9a466422e4ff98857ec3fc2dc9b78 is 50, key is test_row_0/C:col10/1732301880965/Put/seqid=0 2024-11-22T18:58:01,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742311_1487 (size=12001) 2024-11-22T18:58:01,151 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/C/02b9a466422e4ff98857ec3fc2dc9b78 2024-11-22T18:58:01,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/A/e679fead15da4ede900c00a6dcbd7692 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/e679fead15da4ede900c00a6dcbd7692 2024-11-22T18:58:01,167 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/e679fead15da4ede900c00a6dcbd7692, entries=200, sequenceid=16, filesize=38.6 K 2024-11-22T18:58:01,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/B/f0902f4602d64af09204bbd2487010a1 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/f0902f4602d64af09204bbd2487010a1 2024-11-22T18:58:01,173 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/f0902f4602d64af09204bbd2487010a1, entries=150, sequenceid=16, filesize=11.7 K 2024-11-22T18:58:01,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/C/02b9a466422e4ff98857ec3fc2dc9b78 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/02b9a466422e4ff98857ec3fc2dc9b78 2024-11-22T18:58:01,178 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/02b9a466422e4ff98857ec3fc2dc9b78, entries=150, sequenceid=16, filesize=11.7 K 2024-11-22T18:58:01,179 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for cb9c73ed0f14cbdd1f76d53a72aa266b in 212ms, sequenceid=16, compaction requested=false 2024-11-22T18:58:01,179 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cb9c73ed0f14cbdd1f76d53a72aa266b: 2024-11-22T18:58:01,236 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-22T18:58:01,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:01,237 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2902): Flushing cb9c73ed0f14cbdd1f76d53a72aa266b 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-22T18:58:01,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411224cdd0d6c350b40f78dd25f49d0edb927_cb9c73ed0f14cbdd1f76d53a72aa266b is 50, key is test_row_0/A:col10/1732301880985/Put/seqid=0 2024-11-22T18:58:01,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-22T18:58:01,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742312_1488 (size=12154) 2024-11-22T18:58:01,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on cb9c73ed0f14cbdd1f76d53a72aa266b 2024-11-22T18:58:01,318 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. as already flushing 2024-11-22T18:58:01,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:01,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34568 deadline: 1732301941325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:01,331 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:01,331 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:01,331 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:01,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:01,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34578 deadline: 1732301941329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:01,333 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:01,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:01,333 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:01,333 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:01,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34564 deadline: 1732301941330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:01,333 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:01,333 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:01,333 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:01,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:01,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34524 deadline: 1732301941331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:01,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:01,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:01,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1732301941331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:01,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:01,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:01,335 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:01,335 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:01,335 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:01,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:01,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34578 deadline: 1732301941435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:01,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:01,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34564 deadline: 1732301941436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:01,440 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:01,440 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:01,440 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:01,440 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:01,440 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:01,440 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:01,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:01,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34524 deadline: 1732301941446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:01,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:01,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:01,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:01,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:01,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1732301941446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:01,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:01,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:01,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:01,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-22T18:58:01,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:01,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34568 deadline: 1732301941636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:01,638 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:01,639 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:01,639 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:01,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:01,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34578 deadline: 1732301941646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:01,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:01,648 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:01,648 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:01,648 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:01,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34564 deadline: 1732301941646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:01,649 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:01,649 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:01,649 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:01,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:01,655 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411224cdd0d6c350b40f78dd25f49d0edb927_cb9c73ed0f14cbdd1f76d53a72aa266b to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411224cdd0d6c350b40f78dd25f49d0edb927_cb9c73ed0f14cbdd1f76d53a72aa266b 2024-11-22T18:58:01,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/A/d49ace551f5542b8b4c5bfab9e7d7503, store: [table=TestAcidGuarantees family=A region=cb9c73ed0f14cbdd1f76d53a72aa266b] 2024-11-22T18:58:01,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:01,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34524 deadline: 1732301941655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:01,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/A/d49ace551f5542b8b4c5bfab9e7d7503 is 175, key is test_row_0/A:col10/1732301880985/Put/seqid=0 2024-11-22T18:58:01,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:01,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:01,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:01,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1732301941655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:01,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:01,657 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:01,657 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:01,657 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:01,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742313_1489 (size=30955) 2024-11-22T18:58:01,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:01,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34578 deadline: 1732301941956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:01,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:01,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34564 deadline: 1732301941956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:01,958 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:01,958 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:01,958 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:01,958 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:01,959 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:01,959 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:01,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:01,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34524 deadline: 1732301941966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:01,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:01,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:01,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:01,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:01,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1732301941966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:01,969 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:01,969 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:01,970 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:02,061 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/A/d49ace551f5542b8b4c5bfab9e7d7503 2024-11-22T18:58:02,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-22T18:58:02,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/B/2666f98fcf074559a49389cd6ede0d04 is 50, key is test_row_0/B:col10/1732301880985/Put/seqid=0 2024-11-22T18:58:02,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742314_1490 (size=12001) 2024-11-22T18:58:02,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:02,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34568 deadline: 1732301942145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:02,149 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:02,149 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:02,149 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:02,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:02,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34578 deadline: 1732301942465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:02,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:02,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34564 deadline: 1732301942465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:02,468 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:02,468 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:02,468 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:02,468 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:02,468 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:02,469 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:02,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:02,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34524 deadline: 1732301942475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:02,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:02,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:02,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1732301942476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:02,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:02,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:02,479 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:02,479 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:02,479 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:02,480 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/B/2666f98fcf074559a49389cd6ede0d04 2024-11-22T18:58:02,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/C/2d3b222740b74d59830ddc1efd70ed96 is 50, key is test_row_0/C:col10/1732301880985/Put/seqid=0 2024-11-22T18:58:02,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742315_1491 (size=12001) 2024-11-22T18:58:02,525 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-22T18:58:02,900 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/C/2d3b222740b74d59830ddc1efd70ed96 2024-11-22T18:58:02,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/A/d49ace551f5542b8b4c5bfab9e7d7503 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/d49ace551f5542b8b4c5bfab9e7d7503 2024-11-22T18:58:02,907 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/d49ace551f5542b8b4c5bfab9e7d7503, entries=150, sequenceid=41, filesize=30.2 K 2024-11-22T18:58:02,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/B/2666f98fcf074559a49389cd6ede0d04 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/2666f98fcf074559a49389cd6ede0d04 2024-11-22T18:58:02,912 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/2666f98fcf074559a49389cd6ede0d04, entries=150, sequenceid=41, filesize=11.7 K 2024-11-22T18:58:02,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/C/2d3b222740b74d59830ddc1efd70ed96 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/2d3b222740b74d59830ddc1efd70ed96 2024-11-22T18:58:02,919 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/2d3b222740b74d59830ddc1efd70ed96, entries=150, sequenceid=41, filesize=11.7 K 2024-11-22T18:58:02,921 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for cb9c73ed0f14cbdd1f76d53a72aa266b in 1684ms, sequenceid=41, compaction requested=false 2024-11-22T18:58:02,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for cb9c73ed0f14cbdd1f76d53a72aa266b: 2024-11-22T18:58:02,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:02,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-11-22T18:58:02,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-11-22T18:58:02,926 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-22T18:58:02,926 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9950 sec 2024-11-22T18:58:02,932 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 2.0040 sec 2024-11-22T18:58:03,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-22T18:58:03,076 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T18:58:03,076 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T18:58:03,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees 2024-11-22T18:58:03,078 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T18:58:03,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-22T18:58:03,079 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T18:58:03,079 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T18:58:03,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on cb9c73ed0f14cbdd1f76d53a72aa266b 2024-11-22T18:58:03,168 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T18:58:03,168 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cb9c73ed0f14cbdd1f76d53a72aa266b 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-22T18:58:03,177 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122f14f02db921a4e9588dace55fbc56e1c_cb9c73ed0f14cbdd1f76d53a72aa266b is 50, key is test_row_0/A:col10/1732301881317/Put/seqid=0 2024-11-22T18:58:03,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-22T18:58:03,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742316_1492 (size=14594) 2024-11-22T18:58:03,203 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:03,206 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122f14f02db921a4e9588dace55fbc56e1c_cb9c73ed0f14cbdd1f76d53a72aa266b to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122f14f02db921a4e9588dace55fbc56e1c_cb9c73ed0f14cbdd1f76d53a72aa266b 2024-11-22T18:58:03,209 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/A/80f4a74df7a740548a92bc02374ec449, store: [table=TestAcidGuarantees family=A region=cb9c73ed0f14cbdd1f76d53a72aa266b] 2024-11-22T18:58:03,210 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/A/80f4a74df7a740548a92bc02374ec449 is 175, key is test_row_0/A:col10/1732301881317/Put/seqid=0 2024-11-22T18:58:03,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742317_1493 (size=39549) 2024-11-22T18:58:03,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-22T18:58:03,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:03,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. as already flushing 2024-11-22T18:58:03,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:03,231 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:03,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:03,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:03,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:03,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34568 deadline: 1732301943251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:03,256 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:03,257 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:03,257 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:03,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:03,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34568 deadline: 1732301943365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:03,368 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:03,368 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:03,368 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:03,384 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-22T18:58:03,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:03,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. as already flushing 2024-11-22T18:58:03,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:03,385 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:03,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:03,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:03,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-22T18:58:03,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:03,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34578 deadline: 1732301943475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:03,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:03,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34564 deadline: 1732301943476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:03,478 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to d79ba0c344fb,43593,1732301724173 for 'test_row_2' in cb9c73ed0f14cbdd1f76d53a72aa266b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2148 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] ... 32 more 2024-11-22T18:58:03,478 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to d79ba0c344fb,43593,1732301724173 for 'test_row_1' in cb9c73ed0f14cbdd1f76d53a72aa266b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2148 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] ... 32 more 2024-11-22T18:58:03,479 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:03,479 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:03,479 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:03,479 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:03,479 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:03,479 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:03,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:03,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1732301943486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:03,488 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to d79ba0c344fb,43593,1732301724173 for 'test_row_1' in cb9c73ed0f14cbdd1f76d53a72aa266b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] ... 32 more 2024-11-22T18:58:03,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:03,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:03,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:03,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:03,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34524 deadline: 1732301943495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:03,496 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to d79ba0c344fb,43593,1732301724173 for 'test_row_0' in cb9c73ed0f14cbdd1f76d53a72aa266b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] ... 32 more 2024-11-22T18:58:03,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:03,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:03,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:03,537 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-22T18:58:03,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:03,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. as already flushing 2024-11-22T18:58:03,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:03,538 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:03,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:03,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:03,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:03,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34568 deadline: 1732301943576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:03,579 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:03,579 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:03,579 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:03,627 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=54, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/A/80f4a74df7a740548a92bc02374ec449 2024-11-22T18:58:03,634 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/B/2fdf3ea17fe94233b5dae5adb696947d is 50, key is test_row_0/B:col10/1732301881317/Put/seqid=0 2024-11-22T18:58:03,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742318_1494 (size=12001) 2024-11-22T18:58:03,692 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-22T18:58:03,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:03,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. as already flushing 2024-11-22T18:58:03,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:03,692 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:03,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:03,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:03,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-22T18:58:03,845 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-22T18:58:03,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:03,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. as already flushing 2024-11-22T18:58:03,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:03,845 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:03,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:03,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:03,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:03,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34568 deadline: 1732301943885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:03,888 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:03,888 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:03,888 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:03,998 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-22T18:58:03,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:03,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. as already flushing 2024-11-22T18:58:03,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:03,998 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:03,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:03,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:04,043 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/B/2fdf3ea17fe94233b5dae5adb696947d 2024-11-22T18:58:04,050 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/C/cc818d22fb644cbd8872880e5a30c64f is 50, key is test_row_0/C:col10/1732301881317/Put/seqid=0 2024-11-22T18:58:04,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742319_1495 (size=12001) 2024-11-22T18:58:04,151 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-22T18:58:04,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:04,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. as already flushing 2024-11-22T18:58:04,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:04,152 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:04,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:04,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:04,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-22T18:58:04,304 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-22T18:58:04,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:04,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. as already flushing 2024-11-22T18:58:04,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:04,305 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:04,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:04,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:04,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:04,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34568 deadline: 1732301944396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:04,398 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:04,398 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:04,398 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:04,454 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/C/cc818d22fb644cbd8872880e5a30c64f 2024-11-22T18:58:04,456 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-22T18:58:04,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:04,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. as already flushing 2024-11-22T18:58:04,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:04,457 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:04,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:04,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:04,459 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/A/80f4a74df7a740548a92bc02374ec449 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/80f4a74df7a740548a92bc02374ec449 2024-11-22T18:58:04,462 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/80f4a74df7a740548a92bc02374ec449, entries=200, sequenceid=54, filesize=38.6 K 2024-11-22T18:58:04,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/B/2fdf3ea17fe94233b5dae5adb696947d as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/2fdf3ea17fe94233b5dae5adb696947d 2024-11-22T18:58:04,466 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/2fdf3ea17fe94233b5dae5adb696947d, entries=150, sequenceid=54, filesize=11.7 K 2024-11-22T18:58:04,466 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/C/cc818d22fb644cbd8872880e5a30c64f as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/cc818d22fb644cbd8872880e5a30c64f 2024-11-22T18:58:04,469 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/cc818d22fb644cbd8872880e5a30c64f, entries=150, sequenceid=54, filesize=11.7 K 2024-11-22T18:58:04,470 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for cb9c73ed0f14cbdd1f76d53a72aa266b in 1302ms, sequenceid=54, compaction requested=true 2024-11-22T18:58:04,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cb9c73ed0f14cbdd1f76d53a72aa266b: 2024-11-22T18:58:04,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb9c73ed0f14cbdd1f76d53a72aa266b:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T18:58:04,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:58:04,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb9c73ed0f14cbdd1f76d53a72aa266b:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T18:58:04,470 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T18:58:04,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:58:04,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb9c73ed0f14cbdd1f76d53a72aa266b:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T18:58:04,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:58:04,470 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T18:58:04,472 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T18:58:04,472 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1541): cb9c73ed0f14cbdd1f76d53a72aa266b/B is initiating minor compaction (all files) 2024-11-22T18:58:04,472 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb9c73ed0f14cbdd1f76d53a72aa266b/B in TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:04,472 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/f0902f4602d64af09204bbd2487010a1, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/2666f98fcf074559a49389cd6ede0d04, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/2fdf3ea17fe94233b5dae5adb696947d] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp, totalSize=35.2 K 2024-11-22T18:58:04,472 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110053 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T18:58:04,472 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1541): cb9c73ed0f14cbdd1f76d53a72aa266b/A is initiating minor compaction (all files) 2024-11-22T18:58:04,472 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb9c73ed0f14cbdd1f76d53a72aa266b/A in TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:04,472 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/e679fead15da4ede900c00a6dcbd7692, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/d49ace551f5542b8b4c5bfab9e7d7503, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/80f4a74df7a740548a92bc02374ec449] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp, totalSize=107.5 K 2024-11-22T18:58:04,472 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:04,472 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. files: [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/e679fead15da4ede900c00a6dcbd7692, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/d49ace551f5542b8b4c5bfab9e7d7503, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/80f4a74df7a740548a92bc02374ec449] 2024-11-22T18:58:04,473 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting f0902f4602d64af09204bbd2487010a1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732301880965 2024-11-22T18:58:04,473 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting e679fead15da4ede900c00a6dcbd7692, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732301880964 2024-11-22T18:58:04,473 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting d49ace551f5542b8b4c5bfab9e7d7503, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732301880985 2024-11-22T18:58:04,473 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 2666f98fcf074559a49389cd6ede0d04, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732301880985 2024-11-22T18:58:04,480 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80f4a74df7a740548a92bc02374ec449, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732301881317 2024-11-22T18:58:04,480 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 2fdf3ea17fe94233b5dae5adb696947d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732301881317 2024-11-22T18:58:04,490 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cb9c73ed0f14cbdd1f76d53a72aa266b] 2024-11-22T18:58:04,492 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb9c73ed0f14cbdd1f76d53a72aa266b#B#compaction#427 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:58:04,493 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/B/da2146315f5540b9b7bb2d887b04e6cf is 50, key is test_row_0/B:col10/1732301881317/Put/seqid=0 2024-11-22T18:58:04,502 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411221fe69d3cbc54438c84abb8b4e21ddfba_cb9c73ed0f14cbdd1f76d53a72aa266b store=[table=TestAcidGuarantees family=A region=cb9c73ed0f14cbdd1f76d53a72aa266b] 2024-11-22T18:58:04,503 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411221fe69d3cbc54438c84abb8b4e21ddfba_cb9c73ed0f14cbdd1f76d53a72aa266b, store=[table=TestAcidGuarantees family=A region=cb9c73ed0f14cbdd1f76d53a72aa266b] 2024-11-22T18:58:04,504 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411221fe69d3cbc54438c84abb8b4e21ddfba_cb9c73ed0f14cbdd1f76d53a72aa266b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cb9c73ed0f14cbdd1f76d53a72aa266b] 2024-11-22T18:58:04,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742320_1496 (size=12104) 2024-11-22T18:58:04,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742321_1497 (size=4469) 2024-11-22T18:58:04,609 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-22T18:58:04,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:04,610 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2902): Flushing cb9c73ed0f14cbdd1f76d53a72aa266b 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-22T18:58:04,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122e934d832415e4cababf9211a65401528_cb9c73ed0f14cbdd1f76d53a72aa266b is 50, key is test_row_0/A:col10/1732301883246/Put/seqid=0 2024-11-22T18:58:04,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742322_1498 (size=12154) 2024-11-22T18:58:04,909 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/B/da2146315f5540b9b7bb2d887b04e6cf as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/da2146315f5540b9b7bb2d887b04e6cf 2024-11-22T18:58:04,912 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb9c73ed0f14cbdd1f76d53a72aa266b#A#compaction#426 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:58:04,913 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/A/965e04ca1e6a4abe8dd2e06e77e998b6 is 175, key is test_row_0/A:col10/1732301881317/Put/seqid=0 2024-11-22T18:58:04,914 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cb9c73ed0f14cbdd1f76d53a72aa266b/B of cb9c73ed0f14cbdd1f76d53a72aa266b into da2146315f5540b9b7bb2d887b04e6cf(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:58:04,914 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb9c73ed0f14cbdd1f76d53a72aa266b: 2024-11-22T18:58:04,914 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., storeName=cb9c73ed0f14cbdd1f76d53a72aa266b/B, priority=13, startTime=1732301884470; duration=0sec 2024-11-22T18:58:04,914 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:58:04,914 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb9c73ed0f14cbdd1f76d53a72aa266b:B 2024-11-22T18:58:04,914 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T18:58:04,915 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T18:58:04,915 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1541): cb9c73ed0f14cbdd1f76d53a72aa266b/C is initiating minor compaction (all files) 2024-11-22T18:58:04,915 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb9c73ed0f14cbdd1f76d53a72aa266b/C in TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:04,916 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/02b9a466422e4ff98857ec3fc2dc9b78, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/2d3b222740b74d59830ddc1efd70ed96, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/cc818d22fb644cbd8872880e5a30c64f] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp, totalSize=35.2 K 2024-11-22T18:58:04,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742323_1499 (size=31058) 2024-11-22T18:58:04,917 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 02b9a466422e4ff98857ec3fc2dc9b78, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732301880965 2024-11-22T18:58:04,917 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d3b222740b74d59830ddc1efd70ed96, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732301880985 2024-11-22T18:58:04,917 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting cc818d22fb644cbd8872880e5a30c64f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732301881317 2024-11-22T18:58:04,921 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/A/965e04ca1e6a4abe8dd2e06e77e998b6 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/965e04ca1e6a4abe8dd2e06e77e998b6 2024-11-22T18:58:04,925 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb9c73ed0f14cbdd1f76d53a72aa266b#C#compaction#429 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:58:04,925 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/C/1f89af3a5406457aa351ec87a0d7fe65 is 50, key is test_row_0/C:col10/1732301881317/Put/seqid=0 2024-11-22T18:58:04,926 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cb9c73ed0f14cbdd1f76d53a72aa266b/A of cb9c73ed0f14cbdd1f76d53a72aa266b into 965e04ca1e6a4abe8dd2e06e77e998b6(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:58:04,926 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb9c73ed0f14cbdd1f76d53a72aa266b: 2024-11-22T18:58:04,926 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., storeName=cb9c73ed0f14cbdd1f76d53a72aa266b/A, priority=13, startTime=1732301884470; duration=0sec 2024-11-22T18:58:04,926 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:58:04,926 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb9c73ed0f14cbdd1f76d53a72aa266b:A 2024-11-22T18:58:04,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742324_1500 (size=12104) 2024-11-22T18:58:04,933 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/C/1f89af3a5406457aa351ec87a0d7fe65 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/1f89af3a5406457aa351ec87a0d7fe65 2024-11-22T18:58:04,937 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cb9c73ed0f14cbdd1f76d53a72aa266b/C of cb9c73ed0f14cbdd1f76d53a72aa266b into 1f89af3a5406457aa351ec87a0d7fe65(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:58:04,937 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb9c73ed0f14cbdd1f76d53a72aa266b: 2024-11-22T18:58:04,937 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., storeName=cb9c73ed0f14cbdd1f76d53a72aa266b/C, priority=13, startTime=1732301884470; duration=0sec 2024-11-22T18:58:04,937 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:58:04,937 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb9c73ed0f14cbdd1f76d53a72aa266b:C 2024-11-22T18:58:05,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,028 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122e934d832415e4cababf9211a65401528_cb9c73ed0f14cbdd1f76d53a72aa266b to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122e934d832415e4cababf9211a65401528_cb9c73ed0f14cbdd1f76d53a72aa266b 2024-11-22T18:58:05,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/A/531cc4f34d6448d09db4f2ac552b1122, store: [table=TestAcidGuarantees family=A region=cb9c73ed0f14cbdd1f76d53a72aa266b] 2024-11-22T18:58:05,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/A/531cc4f34d6448d09db4f2ac552b1122 is 175, key is test_row_0/A:col10/1732301883246/Put/seqid=0 2024-11-22T18:58:05,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742325_1501 (size=30955) 2024-11-22T18:58:05,036 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=77, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/A/531cc4f34d6448d09db4f2ac552b1122 2024-11-22T18:58:05,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/B/b01171d576ef47209a5d1e973b764bc9 is 50, key is test_row_0/B:col10/1732301883246/Put/seqid=0 2024-11-22T18:58:05,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742326_1502 (size=12001) 2024-11-22T18:58:05,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-22T18:58:05,407 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. as already flushing 2024-11-22T18:58:05,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on cb9c73ed0f14cbdd1f76d53a72aa266b 2024-11-22T18:58:05,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:05,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34568 deadline: 1732301945441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:05,445 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:05,445 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:05,445 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:05,447 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/B/b01171d576ef47209a5d1e973b764bc9 2024-11-22T18:58:05,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/C/d9765992869f4e06bc51ec1401702ddc is 50, key is test_row_0/C:col10/1732301883246/Put/seqid=0 2024-11-22T18:58:05,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742327_1503 (size=12001) 2024-11-22T18:58:05,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:05,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34578 deadline: 1732301945495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:05,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:05,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1732301945495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:05,497 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to d79ba0c344fb,43593,1732301724173 for 'test_row_2' in cb9c73ed0f14cbdd1f76d53a72aa266b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] ... 32 more 2024-11-22T18:58:05,498 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:05,498 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:05,498 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:05,498 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to d79ba0c344fb,43593,1732301724173 for 'test_row_1' in cb9c73ed0f14cbdd1f76d53a72aa266b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] ... 32 more 2024-11-22T18:58:05,498 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:05,498 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:05,498 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:05,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:05,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34564 deadline: 1732301945506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:05,507 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to d79ba0c344fb,43593,1732301724173 for 'test_row_1' in cb9c73ed0f14cbdd1f76d53a72aa266b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] ... 32 more 2024-11-22T18:58:05,508 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:05,508 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:05,508 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:05,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:05,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34524 deadline: 1732301945516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:05,519 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to d79ba0c344fb,43593,1732301724173 for 'test_row_0' in cb9c73ed0f14cbdd1f76d53a72aa266b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] ... 32 more 2024-11-22T18:58:05,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:05,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:05,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:05,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:05,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34568 deadline: 1732301945555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:05,557 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:05,557 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:05,557 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:05,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:05,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34568 deadline: 1732301945765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:05,769 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:05,769 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:05,769 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:05,858 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/C/d9765992869f4e06bc51ec1401702ddc 2024-11-22T18:58:05,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/A/531cc4f34d6448d09db4f2ac552b1122 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/531cc4f34d6448d09db4f2ac552b1122 2024-11-22T18:58:05,867 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/531cc4f34d6448d09db4f2ac552b1122, entries=150, sequenceid=77, filesize=30.2 K 2024-11-22T18:58:05,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/B/b01171d576ef47209a5d1e973b764bc9 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/b01171d576ef47209a5d1e973b764bc9 2024-11-22T18:58:05,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,870 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/b01171d576ef47209a5d1e973b764bc9, entries=150, sequenceid=77, filesize=11.7 K 2024-11-22T18:58:05,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/C/d9765992869f4e06bc51ec1401702ddc as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/d9765992869f4e06bc51ec1401702ddc 2024-11-22T18:58:05,874 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/d9765992869f4e06bc51ec1401702ddc, entries=150, sequenceid=77, filesize=11.7 K 2024-11-22T18:58:05,875 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for cb9c73ed0f14cbdd1f76d53a72aa266b in 1266ms, sequenceid=77, compaction requested=false 2024-11-22T18:58:05,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2603): Flush status journal for cb9c73ed0f14cbdd1f76d53a72aa266b: 2024-11-22T18:58:05,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:05,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=169 2024-11-22T18:58:05,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4169): Remote procedure done, pid=169 2024-11-22T18:58:05,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,878 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-22T18:58:05,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,878 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7970 sec 2024-11-22T18:58:05,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,880 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees in 2.8030 sec 2024-11-22T18:58:05,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:05,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on cb9c73ed0f14cbdd1f76d53a72aa266b 2024-11-22T18:58:06,080 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T18:58:06,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,080 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cb9c73ed0f14cbdd1f76d53a72aa266b 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-22T18:58:06,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,089 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112255b3b44f45e548478f031f33bb3c5374_cb9c73ed0f14cbdd1f76d53a72aa266b is 50, key is test_row_0/A:col10/1732301886079/Put/seqid=0 2024-11-22T18:58:06,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742328_1504 (size=12154) 2024-11-22T18:58:06,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:06,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34568 deadline: 1732301946188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:06,192 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:06,192 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:06,192 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:06,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:06,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34568 deadline: 1732301946295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:06,299 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:06,299 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:06,299 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:06,499 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:06,503 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112255b3b44f45e548478f031f33bb3c5374_cb9c73ed0f14cbdd1f76d53a72aa266b to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112255b3b44f45e548478f031f33bb3c5374_cb9c73ed0f14cbdd1f76d53a72aa266b 2024-11-22T18:58:06,503 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/A/3423d2c747a04471aed42b9eaf138953, store: [table=TestAcidGuarantees family=A region=cb9c73ed0f14cbdd1f76d53a72aa266b] 2024-11-22T18:58:06,504 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/A/3423d2c747a04471aed42b9eaf138953 is 175, key is test_row_0/A:col10/1732301886079/Put/seqid=0 2024-11-22T18:58:06,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742329_1505 (size=30955) 2024-11-22T18:58:06,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:06,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34568 deadline: 1732301946506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:06,509 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:06,509 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:06,509 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:06,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:06,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34568 deadline: 1732301946816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:06,820 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:06,820 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:06,820 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:06,908 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=94, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/A/3423d2c747a04471aed42b9eaf138953 2024-11-22T18:58:06,915 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/B/9b821b53abfc4b2aac21f4017f392d80 is 50, key is test_row_0/B:col10/1732301886079/Put/seqid=0 2024-11-22T18:58:06,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742330_1506 (size=12001) 2024-11-22T18:58:07,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-22T18:58:07,246 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T18:58:07,246 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T18:58:07,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees 2024-11-22T18:58:07,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-22T18:58:07,248 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T18:58:07,249 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T18:58:07,249 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T18:58:07,321 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/B/9b821b53abfc4b2aac21f4017f392d80 2024-11-22T18:58:07,328 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/C/dd84de494dc246f3846e9a5c3bc9b6ed is 50, key is test_row_0/C:col10/1732301886079/Put/seqid=0 2024-11-22T18:58:07,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:07,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34568 deadline: 1732301947326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:07,329 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:07,329 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:07,329 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:07,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742331_1507 (size=12001) 2024-11-22T18:58:07,332 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/C/dd84de494dc246f3846e9a5c3bc9b6ed 2024-11-22T18:58:07,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/A/3423d2c747a04471aed42b9eaf138953 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/3423d2c747a04471aed42b9eaf138953 2024-11-22T18:58:07,339 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/3423d2c747a04471aed42b9eaf138953, entries=150, sequenceid=94, filesize=30.2 K 2024-11-22T18:58:07,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/B/9b821b53abfc4b2aac21f4017f392d80 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/9b821b53abfc4b2aac21f4017f392d80 2024-11-22T18:58:07,343 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/9b821b53abfc4b2aac21f4017f392d80, entries=150, sequenceid=94, filesize=11.7 K 2024-11-22T18:58:07,343 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/C/dd84de494dc246f3846e9a5c3bc9b6ed as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/dd84de494dc246f3846e9a5c3bc9b6ed 2024-11-22T18:58:07,347 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/dd84de494dc246f3846e9a5c3bc9b6ed, entries=150, sequenceid=94, filesize=11.7 K 2024-11-22T18:58:07,348 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for cb9c73ed0f14cbdd1f76d53a72aa266b in 1268ms, sequenceid=94, compaction requested=true 2024-11-22T18:58:07,348 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cb9c73ed0f14cbdd1f76d53a72aa266b: 2024-11-22T18:58:07,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb9c73ed0f14cbdd1f76d53a72aa266b:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T18:58:07,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:58:07,348 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T18:58:07,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb9c73ed0f14cbdd1f76d53a72aa266b:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T18:58:07,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:58:07,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb9c73ed0f14cbdd1f76d53a72aa266b:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T18:58:07,348 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T18:58:07,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:58:07,349 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92968 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T18:58:07,349 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T18:58:07,349 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1541): cb9c73ed0f14cbdd1f76d53a72aa266b/A is initiating minor compaction (all files) 2024-11-22T18:58:07,349 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1541): cb9c73ed0f14cbdd1f76d53a72aa266b/B is initiating minor compaction (all files) 2024-11-22T18:58:07,349 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb9c73ed0f14cbdd1f76d53a72aa266b/A in TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:07,349 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb9c73ed0f14cbdd1f76d53a72aa266b/B in TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:07,349 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/965e04ca1e6a4abe8dd2e06e77e998b6, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/531cc4f34d6448d09db4f2ac552b1122, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/3423d2c747a04471aed42b9eaf138953] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp, totalSize=90.8 K 2024-11-22T18:58:07,349 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/da2146315f5540b9b7bb2d887b04e6cf, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/b01171d576ef47209a5d1e973b764bc9, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/9b821b53abfc4b2aac21f4017f392d80] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp, totalSize=35.3 K 2024-11-22T18:58:07,349 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:07,349 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. files: [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/965e04ca1e6a4abe8dd2e06e77e998b6, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/531cc4f34d6448d09db4f2ac552b1122, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/3423d2c747a04471aed42b9eaf138953] 2024-11-22T18:58:07,349 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 965e04ca1e6a4abe8dd2e06e77e998b6, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732301881317 2024-11-22T18:58:07,349 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting da2146315f5540b9b7bb2d887b04e6cf, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732301881317 2024-11-22T18:58:07,350 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting b01171d576ef47209a5d1e973b764bc9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732301883175 2024-11-22T18:58:07,350 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 531cc4f34d6448d09db4f2ac552b1122, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732301883175 2024-11-22T18:58:07,350 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b821b53abfc4b2aac21f4017f392d80, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732301885437 2024-11-22T18:58:07,350 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3423d2c747a04471aed42b9eaf138953, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732301885437 2024-11-22T18:58:07,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-22T18:58:07,356 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cb9c73ed0f14cbdd1f76d53a72aa266b] 2024-11-22T18:58:07,357 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb9c73ed0f14cbdd1f76d53a72aa266b#B#compaction#435 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:58:07,357 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/B/a405d3ee57604c389a655cede2c646d3 is 50, key is test_row_0/B:col10/1732301886079/Put/seqid=0 2024-11-22T18:58:07,359 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411220dc853bd08324493a02a5abc47bdf1ba_cb9c73ed0f14cbdd1f76d53a72aa266b store=[table=TestAcidGuarantees family=A region=cb9c73ed0f14cbdd1f76d53a72aa266b] 2024-11-22T18:58:07,361 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411220dc853bd08324493a02a5abc47bdf1ba_cb9c73ed0f14cbdd1f76d53a72aa266b, store=[table=TestAcidGuarantees family=A region=cb9c73ed0f14cbdd1f76d53a72aa266b] 2024-11-22T18:58:07,362 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411220dc853bd08324493a02a5abc47bdf1ba_cb9c73ed0f14cbdd1f76d53a72aa266b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cb9c73ed0f14cbdd1f76d53a72aa266b] 2024-11-22T18:58:07,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742332_1508 (size=12207) 2024-11-22T18:58:07,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742333_1509 (size=4469) 2024-11-22T18:58:07,401 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-22T18:58:07,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:07,401 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2902): Flushing cb9c73ed0f14cbdd1f76d53a72aa266b 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-22T18:58:07,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122b8853b4d56d448b7a29c7d1c09e9ba0c_cb9c73ed0f14cbdd1f76d53a72aa266b is 50, key is test_row_0/A:col10/1732301886186/Put/seqid=0 2024-11-22T18:58:07,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742334_1510 (size=12154) 2024-11-22T18:58:07,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-22T18:58:07,767 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/B/a405d3ee57604c389a655cede2c646d3 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/a405d3ee57604c389a655cede2c646d3 2024-11-22T18:58:07,771 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cb9c73ed0f14cbdd1f76d53a72aa266b/B of cb9c73ed0f14cbdd1f76d53a72aa266b into a405d3ee57604c389a655cede2c646d3(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:58:07,772 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb9c73ed0f14cbdd1f76d53a72aa266b: 2024-11-22T18:58:07,772 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., storeName=cb9c73ed0f14cbdd1f76d53a72aa266b/B, priority=13, startTime=1732301887348; duration=0sec 2024-11-22T18:58:07,772 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:58:07,772 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb9c73ed0f14cbdd1f76d53a72aa266b:B 2024-11-22T18:58:07,772 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb9c73ed0f14cbdd1f76d53a72aa266b#A#compaction#436 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:58:07,772 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T18:58:07,772 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/A/45637cefa2cf47e0b946c26182931f17 is 175, key is test_row_0/A:col10/1732301886079/Put/seqid=0 2024-11-22T18:58:07,773 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T18:58:07,773 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1541): cb9c73ed0f14cbdd1f76d53a72aa266b/C is initiating minor compaction (all files) 2024-11-22T18:58:07,773 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb9c73ed0f14cbdd1f76d53a72aa266b/C in TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:07,773 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/1f89af3a5406457aa351ec87a0d7fe65, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/d9765992869f4e06bc51ec1401702ddc, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/dd84de494dc246f3846e9a5c3bc9b6ed] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp, totalSize=35.3 K 2024-11-22T18:58:07,773 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f89af3a5406457aa351ec87a0d7fe65, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732301881317 2024-11-22T18:58:07,774 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting d9765992869f4e06bc51ec1401702ddc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732301883175 2024-11-22T18:58:07,774 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting dd84de494dc246f3846e9a5c3bc9b6ed, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732301885437 2024-11-22T18:58:07,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742335_1511 (size=31161) 2024-11-22T18:58:07,780 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb9c73ed0f14cbdd1f76d53a72aa266b#C#compaction#438 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:58:07,781 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/C/79e4d06e990f42a1b6dea756723d5114 is 50, key is test_row_0/C:col10/1732301886079/Put/seqid=0 2024-11-22T18:58:07,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742336_1512 (size=12207) 2024-11-22T18:58:07,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:07,816 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122b8853b4d56d448b7a29c7d1c09e9ba0c_cb9c73ed0f14cbdd1f76d53a72aa266b to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122b8853b4d56d448b7a29c7d1c09e9ba0c_cb9c73ed0f14cbdd1f76d53a72aa266b 2024-11-22T18:58:07,817 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/C/79e4d06e990f42a1b6dea756723d5114 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/79e4d06e990f42a1b6dea756723d5114 2024-11-22T18:58:07,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/A/af354a9ad25547f1836f382459969358, store: [table=TestAcidGuarantees family=A region=cb9c73ed0f14cbdd1f76d53a72aa266b] 2024-11-22T18:58:07,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/A/af354a9ad25547f1836f382459969358 is 175, key is test_row_0/A:col10/1732301886186/Put/seqid=0 2024-11-22T18:58:07,824 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cb9c73ed0f14cbdd1f76d53a72aa266b/C of cb9c73ed0f14cbdd1f76d53a72aa266b into 79e4d06e990f42a1b6dea756723d5114(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:58:07,824 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb9c73ed0f14cbdd1f76d53a72aa266b: 2024-11-22T18:58:07,824 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., storeName=cb9c73ed0f14cbdd1f76d53a72aa266b/C, priority=13, startTime=1732301887348; duration=0sec 2024-11-22T18:58:07,824 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:58:07,824 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb9c73ed0f14cbdd1f76d53a72aa266b:C 2024-11-22T18:58:07,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742337_1513 (size=30955) 2024-11-22T18:58:07,843 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=116, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/A/af354a9ad25547f1836f382459969358 2024-11-22T18:58:07,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/B/5ffe7da2c2ff44b782672cb1978646c0 is 50, key is test_row_0/B:col10/1732301886186/Put/seqid=0 2024-11-22T18:58:07,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742338_1514 (size=12001) 2024-11-22T18:58:07,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-22T18:58:08,181 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/A/45637cefa2cf47e0b946c26182931f17 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/45637cefa2cf47e0b946c26182931f17 2024-11-22T18:58:08,185 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cb9c73ed0f14cbdd1f76d53a72aa266b/A of cb9c73ed0f14cbdd1f76d53a72aa266b into 45637cefa2cf47e0b946c26182931f17(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:58:08,185 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb9c73ed0f14cbdd1f76d53a72aa266b: 2024-11-22T18:58:08,185 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., storeName=cb9c73ed0f14cbdd1f76d53a72aa266b/A, priority=13, startTime=1732301887348; duration=0sec 2024-11-22T18:58:08,185 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:58:08,185 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb9c73ed0f14cbdd1f76d53a72aa266b:A 2024-11-22T18:58:08,275 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/B/5ffe7da2c2ff44b782672cb1978646c0 2024-11-22T18:58:08,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/C/1c8b70a033b64d25b16884b15ce13417 is 50, key is test_row_0/C:col10/1732301886186/Put/seqid=0 2024-11-22T18:58:08,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742339_1515 (size=12001) 2024-11-22T18:58:08,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on cb9c73ed0f14cbdd1f76d53a72aa266b 2024-11-22T18:58:08,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. as already flushing 2024-11-22T18:58:08,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-22T18:58:08,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:08,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34568 deadline: 1732301948386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:08,390 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:08,390 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:08,390 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:08,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:08,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34568 deadline: 1732301948495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:08,500 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:08,500 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:08,500 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:08,705 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/C/1c8b70a033b64d25b16884b15ce13417 2024-11-22T18:58:08,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:08,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34568 deadline: 1732301948705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:08,708 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:08,708 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:08,708 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:08,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/A/af354a9ad25547f1836f382459969358 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/af354a9ad25547f1836f382459969358 2024-11-22T18:58:08,712 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/af354a9ad25547f1836f382459969358, entries=150, sequenceid=116, filesize=30.2 K 2024-11-22T18:58:08,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/B/5ffe7da2c2ff44b782672cb1978646c0 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/5ffe7da2c2ff44b782672cb1978646c0 2024-11-22T18:58:08,716 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/5ffe7da2c2ff44b782672cb1978646c0, entries=150, sequenceid=116, filesize=11.7 K 2024-11-22T18:58:08,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/C/1c8b70a033b64d25b16884b15ce13417 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/1c8b70a033b64d25b16884b15ce13417 2024-11-22T18:58:08,720 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/1c8b70a033b64d25b16884b15ce13417, entries=150, sequenceid=116, filesize=11.7 K 2024-11-22T18:58:08,721 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for cb9c73ed0f14cbdd1f76d53a72aa266b in 1320ms, sequenceid=116, compaction requested=false 2024-11-22T18:58:08,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2603): Flush status journal for cb9c73ed0f14cbdd1f76d53a72aa266b: 2024-11-22T18:58:08,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:08,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=171 2024-11-22T18:58:08,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4169): Remote procedure done, pid=171 2024-11-22T18:58:08,724 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-11-22T18:58:08,724 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4730 sec 2024-11-22T18:58:08,726 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees in 1.4790 sec 2024-11-22T18:58:09,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on cb9c73ed0f14cbdd1f76d53a72aa266b 2024-11-22T18:58:09,018 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T18:58:09,018 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cb9c73ed0f14cbdd1f76d53a72aa266b 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-22T18:58:09,026 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122b7db3e0f83984124bb647f005405862b_cb9c73ed0f14cbdd1f76d53a72aa266b is 50, key is test_row_0/A:col10/1732301888385/Put/seqid=0 2024-11-22T18:58:09,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742340_1516 (size=12304) 2024-11-22T18:58:09,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:09,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34568 deadline: 1732301949099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:09,103 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:09,103 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:09,103 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:09,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:09,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34568 deadline: 1732301949205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:09,207 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:09,207 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:09,207 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:09,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-22T18:58:09,396 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T18:58:09,396 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T18:58:09,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] procedure2.ProcedureExecutor(1139): Stored pid=172, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees 2024-11-22T18:58:09,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-22T18:58:09,398 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=172, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T18:58:09,399 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=172, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T18:58:09,399 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=173, ppid=172, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T18:58:09,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:09,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34568 deadline: 1732301949416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:09,418 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:09,418 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:09,418 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:09,432 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:09,436 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122b7db3e0f83984124bb647f005405862b_cb9c73ed0f14cbdd1f76d53a72aa266b to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122b7db3e0f83984124bb647f005405862b_cb9c73ed0f14cbdd1f76d53a72aa266b 2024-11-22T18:58:09,437 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/A/bebc5c22d2624e008aa54a68df6c3b6c, store: [table=TestAcidGuarantees family=A region=cb9c73ed0f14cbdd1f76d53a72aa266b] 2024-11-22T18:58:09,437 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/A/bebc5c22d2624e008aa54a68df6c3b6c is 175, key is test_row_0/A:col10/1732301888385/Put/seqid=0 2024-11-22T18:58:09,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742341_1517 (size=31105) 2024-11-22T18:58:09,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-22T18:58:09,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:09,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34578 deadline: 1732301949515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:09,518 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to d79ba0c344fb,43593,1732301724173 for 'test_row_2' in cb9c73ed0f14cbdd1f76d53a72aa266b of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] ... 32 more 2024-11-22T18:58:09,519 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:09,519 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:09,519 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:09,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:09,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1732301949525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:09,529 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to d79ba0c344fb,43593,1732301724173 for 'test_row_1' in cb9c73ed0f14cbdd1f76d53a72aa266b of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] ... 32 more 2024-11-22T18:58:09,530 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:09,530 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:09,530 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:09,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:09,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34564 deadline: 1732301949535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:09,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:09,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34524 deadline: 1732301949536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:09,538 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to d79ba0c344fb,43593,1732301724173 for 'test_row_1' in cb9c73ed0f14cbdd1f76d53a72aa266b of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] ... 32 more 2024-11-22T18:58:09,538 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:09,539 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:09,539 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:09,538 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to d79ba0c344fb,43593,1732301724173 for 'test_row_0' in cb9c73ed0f14cbdd1f76d53a72aa266b of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] ... 32 more 2024-11-22T18:58:09,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:09,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:09,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:09,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-22T18:58:09,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:09,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. as already flushing 2024-11-22T18:58:09,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:09,552 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:09,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:09,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:09,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-22T18:58:09,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:09,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. as already flushing 2024-11-22T18:58:09,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:09,705 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:09,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:09,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:09,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-22T18:58:09,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:09,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34568 deadline: 1732301949725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:09,728 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:09,728 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:09,728 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:09,841 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=134, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/A/bebc5c22d2624e008aa54a68df6c3b6c 2024-11-22T18:58:09,848 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/B/c11e007e2220474bb8fcffc10636145d is 50, key is test_row_0/B:col10/1732301888385/Put/seqid=0 2024-11-22T18:58:09,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742342_1518 (size=12151) 2024-11-22T18:58:09,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-22T18:58:09,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:09,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. as already flushing 2024-11-22T18:58:09,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:09,858 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:09,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:09,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:10,011 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-22T18:58:10,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:10,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. as already flushing 2024-11-22T18:58:10,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:10,011 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:10,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:10,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:10,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-22T18:58:10,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-22T18:58:10,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:10,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. as already flushing 2024-11-22T18:58:10,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:10,165 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:10,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:10,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:10,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:10,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34568 deadline: 1732301950236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:10,240 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:10,240 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:10,240 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:10,251 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/B/c11e007e2220474bb8fcffc10636145d 2024-11-22T18:58:10,258 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/C/93656e82028f4024b64d966f570358da is 50, key is test_row_0/C:col10/1732301888385/Put/seqid=0 2024-11-22T18:58:10,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742343_1519 (size=12151) 2024-11-22T18:58:10,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-22T18:58:10,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:10,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. as already flushing 2024-11-22T18:58:10,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:10,318 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:10,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:10,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:10,475 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-22T18:58:10,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:10,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. as already flushing 2024-11-22T18:58:10,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:10,475 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:10,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:10,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:10,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-22T18:58:10,628 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-22T18:58:10,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:10,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. as already flushing 2024-11-22T18:58:10,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:10,628 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:10,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:10,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:10,662 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/C/93656e82028f4024b64d966f570358da 2024-11-22T18:58:10,666 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/A/bebc5c22d2624e008aa54a68df6c3b6c as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/bebc5c22d2624e008aa54a68df6c3b6c 2024-11-22T18:58:10,670 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/bebc5c22d2624e008aa54a68df6c3b6c, entries=150, sequenceid=134, filesize=30.4 K 2024-11-22T18:58:10,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/B/c11e007e2220474bb8fcffc10636145d as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/c11e007e2220474bb8fcffc10636145d 2024-11-22T18:58:10,674 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/c11e007e2220474bb8fcffc10636145d, entries=150, sequenceid=134, filesize=11.9 K 2024-11-22T18:58:10,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/C/93656e82028f4024b64d966f570358da as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/93656e82028f4024b64d966f570358da 2024-11-22T18:58:10,677 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/93656e82028f4024b64d966f570358da, entries=150, sequenceid=134, filesize=11.9 K 2024-11-22T18:58:10,678 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for cb9c73ed0f14cbdd1f76d53a72aa266b in 1660ms, sequenceid=134, compaction requested=true 2024-11-22T18:58:10,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cb9c73ed0f14cbdd1f76d53a72aa266b: 2024-11-22T18:58:10,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb9c73ed0f14cbdd1f76d53a72aa266b:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T18:58:10,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:58:10,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb9c73ed0f14cbdd1f76d53a72aa266b:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T18:58:10,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:58:10,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb9c73ed0f14cbdd1f76d53a72aa266b:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T18:58:10,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:58:10,678 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T18:58:10,678 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T18:58:10,688 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93221 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T18:58:10,688 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1541): cb9c73ed0f14cbdd1f76d53a72aa266b/A is initiating minor compaction (all files) 2024-11-22T18:58:10,688 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T18:58:10,688 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb9c73ed0f14cbdd1f76d53a72aa266b/A in TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:10,688 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1541): cb9c73ed0f14cbdd1f76d53a72aa266b/B is initiating minor compaction (all files) 2024-11-22T18:58:10,688 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb9c73ed0f14cbdd1f76d53a72aa266b/B in TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:10,689 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/45637cefa2cf47e0b946c26182931f17, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/af354a9ad25547f1836f382459969358, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/bebc5c22d2624e008aa54a68df6c3b6c] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp, totalSize=91.0 K 2024-11-22T18:58:10,689 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:10,689 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/a405d3ee57604c389a655cede2c646d3, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/5ffe7da2c2ff44b782672cb1978646c0, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/c11e007e2220474bb8fcffc10636145d] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp, totalSize=35.5 K 2024-11-22T18:58:10,689 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. files: [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/45637cefa2cf47e0b946c26182931f17, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/af354a9ad25547f1836f382459969358, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/bebc5c22d2624e008aa54a68df6c3b6c] 2024-11-22T18:58:10,689 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting a405d3ee57604c389a655cede2c646d3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732301885437 2024-11-22T18:58:10,689 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45637cefa2cf47e0b946c26182931f17, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732301885437 2024-11-22T18:58:10,689 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting af354a9ad25547f1836f382459969358, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732301886168 2024-11-22T18:58:10,689 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ffe7da2c2ff44b782672cb1978646c0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732301886168 2024-11-22T18:58:10,690 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting c11e007e2220474bb8fcffc10636145d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732301888380 2024-11-22T18:58:10,690 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting bebc5c22d2624e008aa54a68df6c3b6c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732301888380 2024-11-22T18:58:10,696 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cb9c73ed0f14cbdd1f76d53a72aa266b] 2024-11-22T18:58:10,697 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb9c73ed0f14cbdd1f76d53a72aa266b#B#compaction#444 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:58:10,697 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/B/c89bfab893214769bb7e92bf7643e4e6 is 50, key is test_row_0/B:col10/1732301888385/Put/seqid=0 2024-11-22T18:58:10,698 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241122390e1ed038a544bf9feae956d07b64d1_cb9c73ed0f14cbdd1f76d53a72aa266b store=[table=TestAcidGuarantees family=A region=cb9c73ed0f14cbdd1f76d53a72aa266b] 2024-11-22T18:58:10,699 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241122390e1ed038a544bf9feae956d07b64d1_cb9c73ed0f14cbdd1f76d53a72aa266b, store=[table=TestAcidGuarantees family=A region=cb9c73ed0f14cbdd1f76d53a72aa266b] 2024-11-22T18:58:10,699 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122390e1ed038a544bf9feae956d07b64d1_cb9c73ed0f14cbdd1f76d53a72aa266b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cb9c73ed0f14cbdd1f76d53a72aa266b] 2024-11-22T18:58:10,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742344_1520 (size=12459) 2024-11-22T18:58:10,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742345_1521 (size=4469) 2024-11-22T18:58:10,781 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-22T18:58:10,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:10,782 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2902): Flushing cb9c73ed0f14cbdd1f76d53a72aa266b 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-22T18:58:10,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122e76bed5978624b57927cccbba6be55df_cb9c73ed0f14cbdd1f76d53a72aa266b is 50, key is test_row_0/A:col10/1732301889094/Put/seqid=0 2024-11-22T18:58:10,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742346_1522 (size=12304) 2024-11-22T18:58:11,120 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb9c73ed0f14cbdd1f76d53a72aa266b#A#compaction#445 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:58:11,120 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/A/33992c38c538448fbd9949ac6d6161d2 is 175, key is test_row_0/A:col10/1732301888385/Put/seqid=0 2024-11-22T18:58:11,123 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/B/c89bfab893214769bb7e92bf7643e4e6 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/c89bfab893214769bb7e92bf7643e4e6 2024-11-22T18:58:11,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742347_1523 (size=31413) 2024-11-22T18:58:11,127 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cb9c73ed0f14cbdd1f76d53a72aa266b/B of cb9c73ed0f14cbdd1f76d53a72aa266b into c89bfab893214769bb7e92bf7643e4e6(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:58:11,128 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb9c73ed0f14cbdd1f76d53a72aa266b: 2024-11-22T18:58:11,128 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., storeName=cb9c73ed0f14cbdd1f76d53a72aa266b/B, priority=13, startTime=1732301890678; duration=0sec 2024-11-22T18:58:11,128 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:58:11,128 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb9c73ed0f14cbdd1f76d53a72aa266b:B 2024-11-22T18:58:11,128 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T18:58:11,129 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T18:58:11,129 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1541): cb9c73ed0f14cbdd1f76d53a72aa266b/C is initiating minor compaction (all files) 2024-11-22T18:58:11,129 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb9c73ed0f14cbdd1f76d53a72aa266b/C in TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:11,130 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/79e4d06e990f42a1b6dea756723d5114, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/1c8b70a033b64d25b16884b15ce13417, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/93656e82028f4024b64d966f570358da] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp, totalSize=35.5 K 2024-11-22T18:58:11,130 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 79e4d06e990f42a1b6dea756723d5114, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732301885437 2024-11-22T18:58:11,130 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/A/33992c38c538448fbd9949ac6d6161d2 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/33992c38c538448fbd9949ac6d6161d2 2024-11-22T18:58:11,131 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c8b70a033b64d25b16884b15ce13417, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732301886168 2024-11-22T18:58:11,131 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 93656e82028f4024b64d966f570358da, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732301888380 2024-11-22T18:58:11,135 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cb9c73ed0f14cbdd1f76d53a72aa266b/A of cb9c73ed0f14cbdd1f76d53a72aa266b into 33992c38c538448fbd9949ac6d6161d2(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:58:11,135 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb9c73ed0f14cbdd1f76d53a72aa266b: 2024-11-22T18:58:11,135 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., storeName=cb9c73ed0f14cbdd1f76d53a72aa266b/A, priority=13, startTime=1732301890678; duration=0sec 2024-11-22T18:58:11,135 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:58:11,135 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb9c73ed0f14cbdd1f76d53a72aa266b:A 2024-11-22T18:58:11,147 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb9c73ed0f14cbdd1f76d53a72aa266b#C#compaction#447 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:58:11,147 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/C/8d548bcc855646d9b3af0a8cb7efc454 is 50, key is test_row_0/C:col10/1732301888385/Put/seqid=0 2024-11-22T18:58:11,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742348_1524 (size=12459) 2024-11-22T18:58:11,155 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/C/8d548bcc855646d9b3af0a8cb7efc454 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/8d548bcc855646d9b3af0a8cb7efc454 2024-11-22T18:58:11,159 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cb9c73ed0f14cbdd1f76d53a72aa266b/C of cb9c73ed0f14cbdd1f76d53a72aa266b into 8d548bcc855646d9b3af0a8cb7efc454(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:58:11,159 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb9c73ed0f14cbdd1f76d53a72aa266b: 2024-11-22T18:58:11,159 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., storeName=cb9c73ed0f14cbdd1f76d53a72aa266b/C, priority=13, startTime=1732301890678; duration=0sec 2024-11-22T18:58:11,159 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:58:11,159 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb9c73ed0f14cbdd1f76d53a72aa266b:C 2024-11-22T18:58:11,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:11,202 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122e76bed5978624b57927cccbba6be55df_cb9c73ed0f14cbdd1f76d53a72aa266b to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122e76bed5978624b57927cccbba6be55df_cb9c73ed0f14cbdd1f76d53a72aa266b 2024-11-22T18:58:11,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/A/57989b4436df4983a952e0766dd7db07, store: [table=TestAcidGuarantees family=A region=cb9c73ed0f14cbdd1f76d53a72aa266b] 2024-11-22T18:58:11,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/A/57989b4436df4983a952e0766dd7db07 is 175, key is test_row_0/A:col10/1732301889094/Put/seqid=0 2024-11-22T18:58:11,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742349_1525 (size=31105) 2024-11-22T18:58:11,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on cb9c73ed0f14cbdd1f76d53a72aa266b 2024-11-22T18:58:11,259 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. as already flushing 2024-11-22T18:58:11,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:11,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34568 deadline: 1732301951296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:11,300 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:11,300 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:11,300 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:11,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:11,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34568 deadline: 1732301951405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:11,408 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:11,408 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:11,409 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:11,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-22T18:58:11,609 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=155, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/A/57989b4436df4983a952e0766dd7db07 2024-11-22T18:58:11,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/B/b91cbf6785b44fdb88b56b45e8e5b8e7 is 50, key is test_row_0/B:col10/1732301889094/Put/seqid=0 2024-11-22T18:58:11,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:11,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34568 deadline: 1732301951616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:11,620 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:11,620 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:11,620 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:11,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742350_1526 (size=12151) 2024-11-22T18:58:11,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:11,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34568 deadline: 1732301951926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:11,928 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:11,928 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:11,928 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:12,021 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/B/b91cbf6785b44fdb88b56b45e8e5b8e7 2024-11-22T18:58:12,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/C/f5e049c735214bbc84d25627ce62fe2c is 50, key is test_row_0/C:col10/1732301889094/Put/seqid=0 2024-11-22T18:58:12,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742351_1527 (size=12151) 2024-11-22T18:58:12,431 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/C/f5e049c735214bbc84d25627ce62fe2c 2024-11-22T18:58:12,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/A/57989b4436df4983a952e0766dd7db07 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/57989b4436df4983a952e0766dd7db07 2024-11-22T18:58:12,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:12,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34568 deadline: 1732301952435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:12,437 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:12,437 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:12,437 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:12,439 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/57989b4436df4983a952e0766dd7db07, entries=150, sequenceid=155, filesize=30.4 K 2024-11-22T18:58:12,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/B/b91cbf6785b44fdb88b56b45e8e5b8e7 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/b91cbf6785b44fdb88b56b45e8e5b8e7 2024-11-22T18:58:12,443 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/b91cbf6785b44fdb88b56b45e8e5b8e7, entries=150, sequenceid=155, filesize=11.9 K 2024-11-22T18:58:12,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/C/f5e049c735214bbc84d25627ce62fe2c as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/f5e049c735214bbc84d25627ce62fe2c 2024-11-22T18:58:12,447 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/f5e049c735214bbc84d25627ce62fe2c, entries=150, sequenceid=155, filesize=11.9 K 2024-11-22T18:58:12,448 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for cb9c73ed0f14cbdd1f76d53a72aa266b in 1667ms, sequenceid=155, compaction requested=false 2024-11-22T18:58:12,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2603): Flush status journal for cb9c73ed0f14cbdd1f76d53a72aa266b: 2024-11-22T18:58:12,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:12,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=173 2024-11-22T18:58:12,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4169): Remote procedure done, pid=173 2024-11-22T18:58:12,452 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=173, resume processing ppid=172 2024-11-22T18:58:12,452 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=173, ppid=172, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0500 sec 2024-11-22T18:58:12,454 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees in 3.0560 sec 2024-11-22T18:58:13,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on cb9c73ed0f14cbdd1f76d53a72aa266b 2024-11-22T18:58:13,457 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T18:58:13,458 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cb9c73ed0f14cbdd1f76d53a72aa266b 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-22T18:58:13,466 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122d2b00939cd514c50ad1425417dfda49f_cb9c73ed0f14cbdd1f76d53a72aa266b is 50, key is test_row_0/A:col10/1732301891290/Put/seqid=0 2024-11-22T18:58:13,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742352_1528 (size=14794) 2024-11-22T18:58:13,475 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:13,483 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122d2b00939cd514c50ad1425417dfda49f_cb9c73ed0f14cbdd1f76d53a72aa266b to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122d2b00939cd514c50ad1425417dfda49f_cb9c73ed0f14cbdd1f76d53a72aa266b 2024-11-22T18:58:13,484 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/A/a911f63f17f04ff78c63a4b77064477b, store: [table=TestAcidGuarantees family=A region=cb9c73ed0f14cbdd1f76d53a72aa266b] 2024-11-22T18:58:13,485 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/A/a911f63f17f04ff78c63a4b77064477b is 175, key is test_row_0/A:col10/1732301891290/Put/seqid=0 2024-11-22T18:58:13,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742353_1529 (size=39749) 2024-11-22T18:58:13,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:13,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34568 deadline: 1732301953533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:13,538 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:13,538 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:13,538 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:13,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-22T18:58:13,566 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T18:58:13,567 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T18:58:13,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] procedure2.ProcedureExecutor(1139): Stored pid=174, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees 2024-11-22T18:58:13,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-22T18:58:13,568 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=174, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T18:58:13,569 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=174, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T18:58:13,569 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=175, ppid=174, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T18:58:13,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:13,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34568 deadline: 1732301953645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:13,649 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:13,649 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:13,649 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:13,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-22T18:58:13,721 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-22T18:58:13,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:13,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. as already flushing 2024-11-22T18:58:13,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:13,721 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:13,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:13,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:13,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:13,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34568 deadline: 1732301953855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:13,858 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:13,858 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:13,858 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:13,874 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-22T18:58:13,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:13,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. as already flushing 2024-11-22T18:58:13,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:13,874 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:13,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:13,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:13,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-22T18:58:13,891 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=174, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/A/a911f63f17f04ff78c63a4b77064477b 2024-11-22T18:58:13,898 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/B/7de76d9f7c374e51b983abfc78db2ed5 is 50, key is test_row_0/B:col10/1732301891290/Put/seqid=0 2024-11-22T18:58:13,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742354_1530 (size=12151) 2024-11-22T18:58:14,027 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-22T18:58:14,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:14,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. as already flushing 2024-11-22T18:58:14,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:14,027 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:14,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:14,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:14,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:14,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34568 deadline: 1732301954166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:14,169 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:14,169 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:14,169 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:14,180 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-22T18:58:14,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:14,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. as already flushing 2024-11-22T18:58:14,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:14,180 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:14,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:14,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:14,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-22T18:58:14,301 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/B/7de76d9f7c374e51b983abfc78db2ed5 2024-11-22T18:58:14,307 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/C/0ff0d34b533f44b283a52db06472309d is 50, key is test_row_0/C:col10/1732301891290/Put/seqid=0 2024-11-22T18:58:14,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742355_1531 (size=12151) 2024-11-22T18:58:14,333 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-22T18:58:14,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:14,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. as already flushing 2024-11-22T18:58:14,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:14,333 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:14,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:14,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:14,486 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-22T18:58:14,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:14,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. as already flushing 2024-11-22T18:58:14,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:14,486 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:14,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:14,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:14,639 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-22T18:58:14,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:14,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. as already flushing 2024-11-22T18:58:14,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:14,640 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:14,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:14,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:14,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:14,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34568 deadline: 1732301954676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:14,679 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:14,679 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:14,679 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:14,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-22T18:58:14,711 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/C/0ff0d34b533f44b283a52db06472309d 2024-11-22T18:58:14,715 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/A/a911f63f17f04ff78c63a4b77064477b as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/a911f63f17f04ff78c63a4b77064477b 2024-11-22T18:58:14,718 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/a911f63f17f04ff78c63a4b77064477b, entries=200, sequenceid=174, filesize=38.8 K 2024-11-22T18:58:14,718 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/B/7de76d9f7c374e51b983abfc78db2ed5 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/7de76d9f7c374e51b983abfc78db2ed5 2024-11-22T18:58:14,722 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/7de76d9f7c374e51b983abfc78db2ed5, entries=150, sequenceid=174, filesize=11.9 K 2024-11-22T18:58:14,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/C/0ff0d34b533f44b283a52db06472309d as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/0ff0d34b533f44b283a52db06472309d 2024-11-22T18:58:14,730 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/0ff0d34b533f44b283a52db06472309d, entries=150, sequenceid=174, filesize=11.9 K 2024-11-22T18:58:14,730 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for cb9c73ed0f14cbdd1f76d53a72aa266b in 1273ms, sequenceid=174, compaction requested=true 2024-11-22T18:58:14,730 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cb9c73ed0f14cbdd1f76d53a72aa266b: 2024-11-22T18:58:14,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb9c73ed0f14cbdd1f76d53a72aa266b:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T18:58:14,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:58:14,731 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T18:58:14,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb9c73ed0f14cbdd1f76d53a72aa266b:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T18:58:14,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:58:14,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb9c73ed0f14cbdd1f76d53a72aa266b:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T18:58:14,731 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T18:58:14,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:58:14,734 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102267 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T18:58:14,734 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T18:58:14,734 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1541): cb9c73ed0f14cbdd1f76d53a72aa266b/A is initiating minor compaction (all files) 2024-11-22T18:58:14,734 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1541): cb9c73ed0f14cbdd1f76d53a72aa266b/B is initiating minor compaction (all files) 2024-11-22T18:58:14,734 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb9c73ed0f14cbdd1f76d53a72aa266b/B in TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:14,734 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb9c73ed0f14cbdd1f76d53a72aa266b/A in TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:14,734 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/c89bfab893214769bb7e92bf7643e4e6, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/b91cbf6785b44fdb88b56b45e8e5b8e7, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/7de76d9f7c374e51b983abfc78db2ed5] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp, totalSize=35.9 K 2024-11-22T18:58:14,734 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/33992c38c538448fbd9949ac6d6161d2, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/57989b4436df4983a952e0766dd7db07, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/a911f63f17f04ff78c63a4b77064477b] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp, totalSize=99.9 K 2024-11-22T18:58:14,734 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:14,734 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. files: [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/33992c38c538448fbd9949ac6d6161d2, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/57989b4436df4983a952e0766dd7db07, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/a911f63f17f04ff78c63a4b77064477b] 2024-11-22T18:58:14,734 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting c89bfab893214769bb7e92bf7643e4e6, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732301888380 2024-11-22T18:58:14,735 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33992c38c538448fbd9949ac6d6161d2, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732301888380 2024-11-22T18:58:14,735 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting b91cbf6785b44fdb88b56b45e8e5b8e7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732301889084 2024-11-22T18:58:14,735 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57989b4436df4983a952e0766dd7db07, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732301889084 2024-11-22T18:58:14,735 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 7de76d9f7c374e51b983abfc78db2ed5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732301891282 2024-11-22T18:58:14,735 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting a911f63f17f04ff78c63a4b77064477b, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732301891282 2024-11-22T18:58:14,741 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cb9c73ed0f14cbdd1f76d53a72aa266b] 2024-11-22T18:58:14,742 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb9c73ed0f14cbdd1f76d53a72aa266b#B#compaction#453 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:58:14,742 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/B/92c73f41bd6a49d79c50fd4b888e1129 is 50, key is test_row_0/B:col10/1732301891290/Put/seqid=0 2024-11-22T18:58:14,743 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411229aa97b41ba18412f8ea7abb844694d7c_cb9c73ed0f14cbdd1f76d53a72aa266b store=[table=TestAcidGuarantees family=A region=cb9c73ed0f14cbdd1f76d53a72aa266b] 2024-11-22T18:58:14,744 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411229aa97b41ba18412f8ea7abb844694d7c_cb9c73ed0f14cbdd1f76d53a72aa266b, store=[table=TestAcidGuarantees family=A region=cb9c73ed0f14cbdd1f76d53a72aa266b] 2024-11-22T18:58:14,744 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411229aa97b41ba18412f8ea7abb844694d7c_cb9c73ed0f14cbdd1f76d53a72aa266b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cb9c73ed0f14cbdd1f76d53a72aa266b] 2024-11-22T18:58:14,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742356_1532 (size=12561) 2024-11-22T18:58:14,762 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/B/92c73f41bd6a49d79c50fd4b888e1129 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/92c73f41bd6a49d79c50fd4b888e1129 2024-11-22T18:58:14,768 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cb9c73ed0f14cbdd1f76d53a72aa266b/B of cb9c73ed0f14cbdd1f76d53a72aa266b into 92c73f41bd6a49d79c50fd4b888e1129(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:58:14,768 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb9c73ed0f14cbdd1f76d53a72aa266b: 2024-11-22T18:58:14,768 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., storeName=cb9c73ed0f14cbdd1f76d53a72aa266b/B, priority=13, startTime=1732301894731; duration=0sec 2024-11-22T18:58:14,768 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:58:14,768 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb9c73ed0f14cbdd1f76d53a72aa266b:B 2024-11-22T18:58:14,768 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T18:58:14,769 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T18:58:14,769 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1541): cb9c73ed0f14cbdd1f76d53a72aa266b/C is initiating minor compaction (all files) 2024-11-22T18:58:14,769 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb9c73ed0f14cbdd1f76d53a72aa266b/C in TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:14,770 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/8d548bcc855646d9b3af0a8cb7efc454, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/f5e049c735214bbc84d25627ce62fe2c, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/0ff0d34b533f44b283a52db06472309d] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp, totalSize=35.9 K 2024-11-22T18:58:14,771 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d548bcc855646d9b3af0a8cb7efc454, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732301888380 2024-11-22T18:58:14,771 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting f5e049c735214bbc84d25627ce62fe2c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732301889084 2024-11-22T18:58:14,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742357_1533 (size=4469) 2024-11-22T18:58:14,772 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ff0d34b533f44b283a52db06472309d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732301891282 2024-11-22T18:58:14,773 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb9c73ed0f14cbdd1f76d53a72aa266b#A#compaction#454 average throughput is 0.76 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:58:14,774 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/A/e1a88c3b310f420b8af0eac7547d1190 is 175, key is test_row_0/A:col10/1732301891290/Put/seqid=0 2024-11-22T18:58:14,792 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-22T18:58:14,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:14,792 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2902): Flushing cb9c73ed0f14cbdd1f76d53a72aa266b 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-22T18:58:14,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742358_1534 (size=31515) 2024-11-22T18:58:14,800 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/A/e1a88c3b310f420b8af0eac7547d1190 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/e1a88c3b310f420b8af0eac7547d1190 2024-11-22T18:58:14,805 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cb9c73ed0f14cbdd1f76d53a72aa266b/A of cb9c73ed0f14cbdd1f76d53a72aa266b into e1a88c3b310f420b8af0eac7547d1190(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:58:14,805 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb9c73ed0f14cbdd1f76d53a72aa266b: 2024-11-22T18:58:14,805 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., storeName=cb9c73ed0f14cbdd1f76d53a72aa266b/A, priority=13, startTime=1732301894731; duration=0sec 2024-11-22T18:58:14,805 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:58:14,805 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb9c73ed0f14cbdd1f76d53a72aa266b:A 2024-11-22T18:58:14,809 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb9c73ed0f14cbdd1f76d53a72aa266b#C#compaction#455 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:58:14,810 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/C/1e3e0909dcbb4986b26a2dc52f5348fe is 50, key is test_row_0/C:col10/1732301891290/Put/seqid=0 2024-11-22T18:58:14,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742359_1535 (size=12561) 2024-11-22T18:58:14,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122bee51b8f8922410aa21933862d4a9da0_cb9c73ed0f14cbdd1f76d53a72aa266b is 50, key is test_row_0/A:col10/1732301893532/Put/seqid=0 2024-11-22T18:58:14,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742360_1536 (size=12304) 2024-11-22T18:58:14,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:14,842 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122bee51b8f8922410aa21933862d4a9da0_cb9c73ed0f14cbdd1f76d53a72aa266b to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122bee51b8f8922410aa21933862d4a9da0_cb9c73ed0f14cbdd1f76d53a72aa266b 2024-11-22T18:58:14,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/A/1d91fcb19abd4069a0cad811bddc4ce1, store: [table=TestAcidGuarantees family=A region=cb9c73ed0f14cbdd1f76d53a72aa266b] 2024-11-22T18:58:14,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/A/1d91fcb19abd4069a0cad811bddc4ce1 is 175, key is test_row_0/A:col10/1732301893532/Put/seqid=0 2024-11-22T18:58:14,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742361_1537 (size=31105) 2024-11-22T18:58:15,238 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/C/1e3e0909dcbb4986b26a2dc52f5348fe as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/1e3e0909dcbb4986b26a2dc52f5348fe 2024-11-22T18:58:15,243 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cb9c73ed0f14cbdd1f76d53a72aa266b/C of cb9c73ed0f14cbdd1f76d53a72aa266b into 1e3e0909dcbb4986b26a2dc52f5348fe(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:58:15,243 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb9c73ed0f14cbdd1f76d53a72aa266b: 2024-11-22T18:58:15,243 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., storeName=cb9c73ed0f14cbdd1f76d53a72aa266b/C, priority=13, startTime=1732301894731; duration=0sec 2024-11-22T18:58:15,243 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:58:15,243 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb9c73ed0f14cbdd1f76d53a72aa266b:C 2024-11-22T18:58:15,257 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=195, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/A/1d91fcb19abd4069a0cad811bddc4ce1 2024-11-22T18:58:15,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/B/5685e6bb4b9741f4b843365e91e80467 is 50, key is test_row_0/B:col10/1732301893532/Put/seqid=0 2024-11-22T18:58:15,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742362_1538 (size=12151) 2024-11-22T18:58:15,672 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/B/5685e6bb4b9741f4b843365e91e80467 2024-11-22T18:58:15,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/C/6c267aee58404e259b2077c8169b0344 is 50, key is test_row_0/C:col10/1732301893532/Put/seqid=0 2024-11-22T18:58:15,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742363_1539 (size=12151) 2024-11-22T18:58:15,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on cb9c73ed0f14cbdd1f76d53a72aa266b 2024-11-22T18:58:15,688 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. as already flushing 2024-11-22T18:58:15,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-22T18:58:15,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:15,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34568 deadline: 1732301955740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:15,744 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:15,744 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:15,744 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:15,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:15,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34568 deadline: 1732301955855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:15,858 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:15,858 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:15,858 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:16,068 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:16,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34568 deadline: 1732301956066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:16,069 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:16,069 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:16,069 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:16,083 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/C/6c267aee58404e259b2077c8169b0344 2024-11-22T18:58:16,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/A/1d91fcb19abd4069a0cad811bddc4ce1 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/1d91fcb19abd4069a0cad811bddc4ce1 2024-11-22T18:58:16,090 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/1d91fcb19abd4069a0cad811bddc4ce1, entries=150, sequenceid=195, filesize=30.4 K 2024-11-22T18:58:16,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/B/5685e6bb4b9741f4b843365e91e80467 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/5685e6bb4b9741f4b843365e91e80467 2024-11-22T18:58:16,094 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/5685e6bb4b9741f4b843365e91e80467, entries=150, sequenceid=195, filesize=11.9 K 2024-11-22T18:58:16,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/C/6c267aee58404e259b2077c8169b0344 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/6c267aee58404e259b2077c8169b0344 2024-11-22T18:58:16,097 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/6c267aee58404e259b2077c8169b0344, entries=150, sequenceid=195, filesize=11.9 K 2024-11-22T18:58:16,098 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for cb9c73ed0f14cbdd1f76d53a72aa266b in 1306ms, sequenceid=195, compaction requested=false 2024-11-22T18:58:16,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2603): Flush status journal for cb9c73ed0f14cbdd1f76d53a72aa266b: 2024-11-22T18:58:16,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:16,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=175 2024-11-22T18:58:16,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4169): Remote procedure done, pid=175 2024-11-22T18:58:16,101 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=175, resume processing ppid=174 2024-11-22T18:58:16,101 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=175, ppid=174, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5300 sec 2024-11-22T18:58:16,103 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=174, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees in 2.5350 sec 2024-11-22T18:58:16,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on cb9c73ed0f14cbdd1f76d53a72aa266b 2024-11-22T18:58:16,378 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T18:58:16,378 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cb9c73ed0f14cbdd1f76d53a72aa266b 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-22T18:58:16,386 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122702198abbfe34a3ca02d2f6a517c2164_cb9c73ed0f14cbdd1f76d53a72aa266b is 50, key is test_row_0/A:col10/1732301896377/Put/seqid=0 2024-11-22T18:58:16,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742364_1540 (size=14794) 2024-11-22T18:58:16,403 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:16,406 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122702198abbfe34a3ca02d2f6a517c2164_cb9c73ed0f14cbdd1f76d53a72aa266b to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122702198abbfe34a3ca02d2f6a517c2164_cb9c73ed0f14cbdd1f76d53a72aa266b 2024-11-22T18:58:16,407 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/A/916c2bddf8594497a82d9e5408dc02e3, store: [table=TestAcidGuarantees family=A region=cb9c73ed0f14cbdd1f76d53a72aa266b] 2024-11-22T18:58:16,408 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/A/916c2bddf8594497a82d9e5408dc02e3 is 175, key is test_row_0/A:col10/1732301896377/Put/seqid=0 2024-11-22T18:58:16,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742365_1541 (size=39749) 2024-11-22T18:58:16,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:16,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34568 deadline: 1732301956450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:16,454 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:16,454 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:16,454 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:16,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:16,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34568 deadline: 1732301956566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:16,569 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:16,569 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:16,569 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:16,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:16,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34568 deadline: 1732301956775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:16,778 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:16,778 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:16,778 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:16,819 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=214, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/A/916c2bddf8594497a82d9e5408dc02e3 2024-11-22T18:58:16,828 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/B/07dfcd11decf4500843e7e96919c9f6b is 50, key is test_row_0/B:col10/1732301896377/Put/seqid=0 2024-11-22T18:58:16,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742366_1542 (size=12151) 2024-11-22T18:58:16,832 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/B/07dfcd11decf4500843e7e96919c9f6b 2024-11-22T18:58:16,842 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/C/216ec2b48ed3447e954ee0ccb5aa44f6 is 50, key is test_row_0/C:col10/1732301896377/Put/seqid=0 2024-11-22T18:58:16,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742367_1543 (size=12151) 2024-11-22T18:58:17,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:17,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34568 deadline: 1732301957085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:17,088 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:17,088 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:17,088 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:17,251 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/C/216ec2b48ed3447e954ee0ccb5aa44f6 2024-11-22T18:58:17,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/A/916c2bddf8594497a82d9e5408dc02e3 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/916c2bddf8594497a82d9e5408dc02e3 2024-11-22T18:58:17,258 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/916c2bddf8594497a82d9e5408dc02e3, entries=200, sequenceid=214, filesize=38.8 K 2024-11-22T18:58:17,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/B/07dfcd11decf4500843e7e96919c9f6b as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/07dfcd11decf4500843e7e96919c9f6b 2024-11-22T18:58:17,262 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/07dfcd11decf4500843e7e96919c9f6b, entries=150, sequenceid=214, filesize=11.9 K 2024-11-22T18:58:17,262 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/C/216ec2b48ed3447e954ee0ccb5aa44f6 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/216ec2b48ed3447e954ee0ccb5aa44f6 2024-11-22T18:58:17,265 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/216ec2b48ed3447e954ee0ccb5aa44f6, entries=150, sequenceid=214, filesize=11.9 K 2024-11-22T18:58:17,266 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for cb9c73ed0f14cbdd1f76d53a72aa266b in 888ms, sequenceid=214, compaction requested=true 2024-11-22T18:58:17,266 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cb9c73ed0f14cbdd1f76d53a72aa266b: 2024-11-22T18:58:17,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb9c73ed0f14cbdd1f76d53a72aa266b:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T18:58:17,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:58:17,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb9c73ed0f14cbdd1f76d53a72aa266b:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T18:58:17,266 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T18:58:17,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:58:17,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb9c73ed0f14cbdd1f76d53a72aa266b:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T18:58:17,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:58:17,266 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T18:58:17,267 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T18:58:17,267 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102369 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T18:58:17,267 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1541): cb9c73ed0f14cbdd1f76d53a72aa266b/A is initiating minor compaction (all files) 2024-11-22T18:58:17,267 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1541): cb9c73ed0f14cbdd1f76d53a72aa266b/B is initiating minor compaction (all files) 2024-11-22T18:58:17,267 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb9c73ed0f14cbdd1f76d53a72aa266b/A in TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:17,267 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb9c73ed0f14cbdd1f76d53a72aa266b/B in TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:17,267 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/92c73f41bd6a49d79c50fd4b888e1129, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/5685e6bb4b9741f4b843365e91e80467, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/07dfcd11decf4500843e7e96919c9f6b] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp, totalSize=36.0 K 2024-11-22T18:58:17,267 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/e1a88c3b310f420b8af0eac7547d1190, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/1d91fcb19abd4069a0cad811bddc4ce1, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/916c2bddf8594497a82d9e5408dc02e3] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp, totalSize=100.0 K 2024-11-22T18:58:17,268 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:17,268 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. files: [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/e1a88c3b310f420b8af0eac7547d1190, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/1d91fcb19abd4069a0cad811bddc4ce1, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/916c2bddf8594497a82d9e5408dc02e3] 2024-11-22T18:58:17,268 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 92c73f41bd6a49d79c50fd4b888e1129, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732301891282 2024-11-22T18:58:17,268 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1a88c3b310f420b8af0eac7547d1190, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732301891282 2024-11-22T18:58:17,269 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 5685e6bb4b9741f4b843365e91e80467, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732301893517 2024-11-22T18:58:17,269 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d91fcb19abd4069a0cad811bddc4ce1, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732301893517 2024-11-22T18:58:17,269 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 07dfcd11decf4500843e7e96919c9f6b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732301895718 2024-11-22T18:58:17,269 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 916c2bddf8594497a82d9e5408dc02e3, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732301895718 2024-11-22T18:58:17,277 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cb9c73ed0f14cbdd1f76d53a72aa266b] 2024-11-22T18:58:17,278 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb9c73ed0f14cbdd1f76d53a72aa266b#B#compaction#463 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:58:17,279 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/B/569b558e605c4750a38e0ca52e422350 is 50, key is test_row_0/B:col10/1732301896377/Put/seqid=0 2024-11-22T18:58:17,279 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411229f684aa64a7f4a15a35b36a4f0de1a5e_cb9c73ed0f14cbdd1f76d53a72aa266b store=[table=TestAcidGuarantees family=A region=cb9c73ed0f14cbdd1f76d53a72aa266b] 2024-11-22T18:58:17,281 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411229f684aa64a7f4a15a35b36a4f0de1a5e_cb9c73ed0f14cbdd1f76d53a72aa266b, store=[table=TestAcidGuarantees family=A region=cb9c73ed0f14cbdd1f76d53a72aa266b] 2024-11-22T18:58:17,281 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411229f684aa64a7f4a15a35b36a4f0de1a5e_cb9c73ed0f14cbdd1f76d53a72aa266b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cb9c73ed0f14cbdd1f76d53a72aa266b] 2024-11-22T18:58:17,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742368_1544 (size=12663) 2024-11-22T18:58:17,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742369_1545 (size=4469) 2024-11-22T18:58:17,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on cb9c73ed0f14cbdd1f76d53a72aa266b 2024-11-22T18:58:17,597 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T18:58:17,597 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cb9c73ed0f14cbdd1f76d53a72aa266b 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-22T18:58:17,604 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122188cebeaad794cc295d2666ab042407e_cb9c73ed0f14cbdd1f76d53a72aa266b is 50, key is test_row_0/A:col10/1732301897596/Put/seqid=0 2024-11-22T18:58:17,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742370_1546 (size=14794) 2024-11-22T18:58:17,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:17,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34568 deadline: 1732301957661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:17,665 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:17,665 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:17,665 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:17,687 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb9c73ed0f14cbdd1f76d53a72aa266b#A#compaction#462 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:58:17,688 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/B/569b558e605c4750a38e0ca52e422350 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/569b558e605c4750a38e0ca52e422350 2024-11-22T18:58:17,688 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/A/567c8d1e8aff4996b0eea1e37dc56698 is 175, key is test_row_0/A:col10/1732301896377/Put/seqid=0 2024-11-22T18:58:17,695 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cb9c73ed0f14cbdd1f76d53a72aa266b/B of cb9c73ed0f14cbdd1f76d53a72aa266b into 569b558e605c4750a38e0ca52e422350(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:58:17,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742371_1547 (size=31617) 2024-11-22T18:58:17,696 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb9c73ed0f14cbdd1f76d53a72aa266b: 2024-11-22T18:58:17,696 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., storeName=cb9c73ed0f14cbdd1f76d53a72aa266b/B, priority=13, startTime=1732301897266; duration=0sec 2024-11-22T18:58:17,696 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:58:17,696 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb9c73ed0f14cbdd1f76d53a72aa266b:B 2024-11-22T18:58:17,696 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T18:58:17,697 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T18:58:17,697 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1541): cb9c73ed0f14cbdd1f76d53a72aa266b/C is initiating minor compaction (all files) 2024-11-22T18:58:17,697 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb9c73ed0f14cbdd1f76d53a72aa266b/C in TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:17,698 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/1e3e0909dcbb4986b26a2dc52f5348fe, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/6c267aee58404e259b2077c8169b0344, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/216ec2b48ed3447e954ee0ccb5aa44f6] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp, totalSize=36.0 K 2024-11-22T18:58:17,698 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e3e0909dcbb4986b26a2dc52f5348fe, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732301891282 2024-11-22T18:58:17,699 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c267aee58404e259b2077c8169b0344, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732301893517 2024-11-22T18:58:17,699 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 216ec2b48ed3447e954ee0ccb5aa44f6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732301895718 2024-11-22T18:58:17,701 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/A/567c8d1e8aff4996b0eea1e37dc56698 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/567c8d1e8aff4996b0eea1e37dc56698 2024-11-22T18:58:17,707 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb9c73ed0f14cbdd1f76d53a72aa266b#C#compaction#465 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:58:17,708 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cb9c73ed0f14cbdd1f76d53a72aa266b/A of cb9c73ed0f14cbdd1f76d53a72aa266b into 567c8d1e8aff4996b0eea1e37dc56698(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:58:17,708 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb9c73ed0f14cbdd1f76d53a72aa266b: 2024-11-22T18:58:17,708 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., storeName=cb9c73ed0f14cbdd1f76d53a72aa266b/A, priority=13, startTime=1732301897266; duration=0sec 2024-11-22T18:58:17,708 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:58:17,708 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb9c73ed0f14cbdd1f76d53a72aa266b:A 2024-11-22T18:58:17,708 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/C/73f7cb21d98042e1b8b28ccb2abff049 is 50, key is test_row_0/C:col10/1732301896377/Put/seqid=0 2024-11-22T18:58:17,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742372_1548 (size=12663) 2024-11-22T18:58:17,717 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/C/73f7cb21d98042e1b8b28ccb2abff049 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/73f7cb21d98042e1b8b28ccb2abff049 2024-11-22T18:58:17,724 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cb9c73ed0f14cbdd1f76d53a72aa266b/C of cb9c73ed0f14cbdd1f76d53a72aa266b into 73f7cb21d98042e1b8b28ccb2abff049(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:58:17,724 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb9c73ed0f14cbdd1f76d53a72aa266b: 2024-11-22T18:58:17,725 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., storeName=cb9c73ed0f14cbdd1f76d53a72aa266b/C, priority=13, startTime=1732301897266; duration=0sec 2024-11-22T18:58:17,725 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:58:17,725 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb9c73ed0f14cbdd1f76d53a72aa266b:C 2024-11-22T18:58:17,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-22T18:58:17,746 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T18:58:17,747 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T18:58:17,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] procedure2.ProcedureExecutor(1139): Stored pid=176, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=176, table=TestAcidGuarantees 2024-11-22T18:58:17,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-22T18:58:17,749 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=176, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=176, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T18:58:17,750 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=176, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=176, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T18:58:17,750 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=177, ppid=176, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T18:58:17,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:17,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34568 deadline: 1732301957775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:17,779 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:17,779 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:17,779 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:17,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-22T18:58:17,903 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=177 2024-11-22T18:58:17,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:17,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. as already flushing 2024-11-22T18:58:17,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:17,903 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] handler.RSProcedureHandler(58): pid=177 java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:17,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=177 java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:17,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=177 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:17,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:17,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34568 deadline: 1732301957985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:17,990 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:17,990 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:17,990 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:18,013 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:18,017 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122188cebeaad794cc295d2666ab042407e_cb9c73ed0f14cbdd1f76d53a72aa266b to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122188cebeaad794cc295d2666ab042407e_cb9c73ed0f14cbdd1f76d53a72aa266b 2024-11-22T18:58:18,019 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/A/0a28d7a403c54b6c8a6f7beeb856cc53, store: [table=TestAcidGuarantees family=A region=cb9c73ed0f14cbdd1f76d53a72aa266b] 2024-11-22T18:58:18,020 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/A/0a28d7a403c54b6c8a6f7beeb856cc53 is 175, key is test_row_0/A:col10/1732301897596/Put/seqid=0 2024-11-22T18:58:18,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742373_1549 (size=39749) 2024-11-22T18:58:18,025 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=234, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/A/0a28d7a403c54b6c8a6f7beeb856cc53 2024-11-22T18:58:18,033 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/B/bfac8c50acdf4fd9ad8392ac65566806 is 50, key is test_row_0/B:col10/1732301897596/Put/seqid=0 2024-11-22T18:58:18,056 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=177 2024-11-22T18:58:18,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:18,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. as already flushing 2024-11-22T18:58:18,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:18,057 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] handler.RSProcedureHandler(58): pid=177 java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:18,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=177 java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:18,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=177 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:18,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742374_1550 (size=12151) 2024-11-22T18:58:18,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-22T18:58:18,210 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=177 2024-11-22T18:58:18,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:18,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. as already flushing 2024-11-22T18:58:18,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:18,211 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] handler.RSProcedureHandler(58): pid=177 java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:18,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=177 java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:18,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=177 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:18,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:18,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34568 deadline: 1732301958295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:18,299 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:18,299 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:18,299 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:18,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=177 2024-11-22T18:58:18,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:18,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. as already flushing 2024-11-22T18:58:18,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:18,364 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] handler.RSProcedureHandler(58): pid=177 java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:18,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=177 java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:18,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=177 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:18,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-22T18:58:18,467 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/B/bfac8c50acdf4fd9ad8392ac65566806 2024-11-22T18:58:18,474 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/C/9d50640af9d9444a9500360b225adaa1 is 50, key is test_row_0/C:col10/1732301897596/Put/seqid=0 2024-11-22T18:58:18,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742375_1551 (size=12151) 2024-11-22T18:58:18,477 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/C/9d50640af9d9444a9500360b225adaa1 2024-11-22T18:58:18,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/A/0a28d7a403c54b6c8a6f7beeb856cc53 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/0a28d7a403c54b6c8a6f7beeb856cc53 2024-11-22T18:58:18,486 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/0a28d7a403c54b6c8a6f7beeb856cc53, entries=200, sequenceid=234, filesize=38.8 K 2024-11-22T18:58:18,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/B/bfac8c50acdf4fd9ad8392ac65566806 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/bfac8c50acdf4fd9ad8392ac65566806 2024-11-22T18:58:18,490 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/bfac8c50acdf4fd9ad8392ac65566806, entries=150, sequenceid=234, filesize=11.9 K 2024-11-22T18:58:18,491 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/C/9d50640af9d9444a9500360b225adaa1 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/9d50640af9d9444a9500360b225adaa1 2024-11-22T18:58:18,495 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/9d50640af9d9444a9500360b225adaa1, entries=150, sequenceid=234, filesize=11.9 K 2024-11-22T18:58:18,496 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for cb9c73ed0f14cbdd1f76d53a72aa266b in 899ms, sequenceid=234, compaction requested=false 2024-11-22T18:58:18,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cb9c73ed0f14cbdd1f76d53a72aa266b: 2024-11-22T18:58:18,516 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=177 2024-11-22T18:58:18,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:18,517 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(2902): Flushing cb9c73ed0f14cbdd1f76d53a72aa266b 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-22T18:58:18,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122f9b600011b324dbcbf1eff85a4a15104_cb9c73ed0f14cbdd1f76d53a72aa266b is 50, key is test_row_0/A:col10/1732301897660/Put/seqid=0 2024-11-22T18:58:18,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742376_1552 (size=12304) 2024-11-22T18:58:18,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:18,530 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122f9b600011b324dbcbf1eff85a4a15104_cb9c73ed0f14cbdd1f76d53a72aa266b to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122f9b600011b324dbcbf1eff85a4a15104_cb9c73ed0f14cbdd1f76d53a72aa266b 2024-11-22T18:58:18,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/A/098ee73c21d440feb9ecdd77c759ebfe, store: [table=TestAcidGuarantees family=A region=cb9c73ed0f14cbdd1f76d53a72aa266b] 2024-11-22T18:58:18,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/A/098ee73c21d440feb9ecdd77c759ebfe is 175, key is test_row_0/A:col10/1732301897660/Put/seqid=0 2024-11-22T18:58:18,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742377_1553 (size=31105) 2024-11-22T18:58:18,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on cb9c73ed0f14cbdd1f76d53a72aa266b 2024-11-22T18:58:18,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. as already flushing 2024-11-22T18:58:18,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:18,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34568 deadline: 1732301958875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:18,879 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:18,879 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:18,879 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:18,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-22T18:58:18,935 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=253, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/A/098ee73c21d440feb9ecdd77c759ebfe 2024-11-22T18:58:18,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/B/0754853085a1462bb1d1d5a21b4603ab is 50, key is test_row_0/B:col10/1732301897660/Put/seqid=0 2024-11-22T18:58:18,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742378_1554 (size=12151) 2024-11-22T18:58:18,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:18,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34568 deadline: 1732301958985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:18,989 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:18,989 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:18,989 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:19,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:19,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34568 deadline: 1732301959195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:19,198 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:19,198 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:19,198 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:19,345 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/B/0754853085a1462bb1d1d5a21b4603ab 2024-11-22T18:58:19,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/C/7ca0114f5f274ccaa66c25b4997089cd is 50, key is test_row_0/C:col10/1732301897660/Put/seqid=0 2024-11-22T18:58:19,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742379_1555 (size=12151) 2024-11-22T18:58:19,356 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/C/7ca0114f5f274ccaa66c25b4997089cd 2024-11-22T18:58:19,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/A/098ee73c21d440feb9ecdd77c759ebfe as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/098ee73c21d440feb9ecdd77c759ebfe 2024-11-22T18:58:19,363 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/098ee73c21d440feb9ecdd77c759ebfe, entries=150, sequenceid=253, filesize=30.4 K 2024-11-22T18:58:19,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/B/0754853085a1462bb1d1d5a21b4603ab as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/0754853085a1462bb1d1d5a21b4603ab 2024-11-22T18:58:19,366 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/0754853085a1462bb1d1d5a21b4603ab, entries=150, sequenceid=253, filesize=11.9 K 2024-11-22T18:58:19,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/C/7ca0114f5f274ccaa66c25b4997089cd as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/7ca0114f5f274ccaa66c25b4997089cd 2024-11-22T18:58:19,370 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/7ca0114f5f274ccaa66c25b4997089cd, entries=150, sequenceid=253, filesize=11.9 K 2024-11-22T18:58:19,371 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for cb9c73ed0f14cbdd1f76d53a72aa266b in 854ms, sequenceid=253, compaction requested=true 2024-11-22T18:58:19,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(2603): Flush status journal for cb9c73ed0f14cbdd1f76d53a72aa266b: 2024-11-22T18:58:19,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:19,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=177 2024-11-22T18:58:19,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4169): Remote procedure done, pid=177 2024-11-22T18:58:19,374 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=177, resume processing ppid=176 2024-11-22T18:58:19,374 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=177, ppid=176, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6220 sec 2024-11-22T18:58:19,375 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=176, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=176, table=TestAcidGuarantees in 1.6270 sec 2024-11-22T18:58:19,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on cb9c73ed0f14cbdd1f76d53a72aa266b 2024-11-22T18:58:19,508 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-22T18:58:19,508 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cb9c73ed0f14cbdd1f76d53a72aa266b 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-22T18:58:19,514 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122731106beece44fd1a59fcbc66be65b71_cb9c73ed0f14cbdd1f76d53a72aa266b is 50, key is test_row_0/A:col10/1732301899507/Put/seqid=0 2024-11-22T18:58:19,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742380_1556 (size=14994) 2024-11-22T18:58:19,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:19,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34524 deadline: 1732301959556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:19,561 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to d79ba0c344fb,43593,1732301724173 for 'test_row_0' in cb9c73ed0f14cbdd1f76d53a72aa266b of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18230 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] ... 32 more 2024-11-22T18:58:19,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:19,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:19,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:19,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:19,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34568 deadline: 1732301959560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:19,565 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:19,565 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:19,565 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:19,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:19,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34564 deadline: 1732301959566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:19,569 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to d79ba0c344fb,43593,1732301724173 for 'test_row_1' in cb9c73ed0f14cbdd1f76d53a72aa266b of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18239 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] ... 32 more 2024-11-22T18:58:19,569 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:19,569 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:19,569 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:19,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:19,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34536 deadline: 1732301959615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:19,619 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to d79ba0c344fb,43593,1732301724173 for 'test_row_1' in cb9c73ed0f14cbdd1f76d53a72aa266b of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18288 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] ... 32 more 2024-11-22T18:58:19,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:19,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:19,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:19,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:19,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34578 deadline: 1732301959625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:19,629 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to d79ba0c344fb,43593,1732301724173 for 'test_row_2' in cb9c73ed0f14cbdd1f76d53a72aa266b of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18300 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] ... 32 more 2024-11-22T18:58:19,630 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:19,630 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:19,630 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:19,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:19,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34568 deadline: 1732301959675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:19,678 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:19,678 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:19,678 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:19,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:19,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34568 deadline: 1732301959886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:19,887 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:19,887 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:19,887 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:19,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-22T18:58:19,896 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T18:58:19,896 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T18:58:19,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] procedure2.ProcedureExecutor(1139): Stored pid=178, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=178, table=TestAcidGuarantees 2024-11-22T18:58:19,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-11-22T18:58:19,898 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=178, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=178, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T18:58:19,899 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=178, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=178, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T18:58:19,899 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=179, ppid=178, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T18:58:19,918 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:19,921 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122731106beece44fd1a59fcbc66be65b71_cb9c73ed0f14cbdd1f76d53a72aa266b to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122731106beece44fd1a59fcbc66be65b71_cb9c73ed0f14cbdd1f76d53a72aa266b 2024-11-22T18:58:19,922 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/A/8bfce39253124ec4af3b9211c3e5b316, store: [table=TestAcidGuarantees family=A region=cb9c73ed0f14cbdd1f76d53a72aa266b] 2024-11-22T18:58:19,922 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/A/8bfce39253124ec4af3b9211c3e5b316 is 175, key is test_row_0/A:col10/1732301899507/Put/seqid=0 2024-11-22T18:58:19,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742381_1557 (size=39949) 2024-11-22T18:58:20,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-11-22T18:58:20,051 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=179 2024-11-22T18:58:20,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:20,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. as already flushing 2024-11-22T18:58:20,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:20,051 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] handler.RSProcedureHandler(58): pid=179 java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:20,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=179 java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:20,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=179 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:20,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:20,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 244 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34568 deadline: 1732301960195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:20,200 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:20,200 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:20,200 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:20,204 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=179 2024-11-22T18:58:20,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:20,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. as already flushing 2024-11-22T18:58:20,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:20,204 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] handler.RSProcedureHandler(58): pid=179 java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:20,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=179 java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:20,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=179 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:20,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-11-22T18:58:20,326 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=274, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/A/8bfce39253124ec4af3b9211c3e5b316 2024-11-22T18:58:20,333 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/B/9f2b4e66a88c4404a8a3ac74ccf08f87 is 50, key is test_row_0/B:col10/1732301899507/Put/seqid=0 2024-11-22T18:58:20,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742382_1558 (size=12301) 2024-11-22T18:58:20,357 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=179 2024-11-22T18:58:20,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:20,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. as already flushing 2024-11-22T18:58:20,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:20,357 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] handler.RSProcedureHandler(58): pid=179 java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:20,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=179 java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:20,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=179 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:20,509 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=179 2024-11-22T18:58:20,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:20,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. as already flushing 2024-11-22T18:58:20,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:20,510 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] handler.RSProcedureHandler(58): pid=179 java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:20,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=179 java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:20,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=179 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:20,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-11-22T18:58:20,662 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=179 2024-11-22T18:58:20,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:20,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. as already flushing 2024-11-22T18:58:20,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:20,663 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] handler.RSProcedureHandler(58): pid=179 java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:20,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=179 java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:20,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=179 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:20,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T18:58:20,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] ipc.CallRunner(138): callId: 245 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34568 deadline: 1732301960706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:20,709 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 , the old value is region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:20,709 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T18:58:20,709 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., hostname=d79ba0c344fb,43593,1732301724173, seqNum=5 because the exception is null or not the one we care about 2024-11-22T18:58:20,737 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/B/9f2b4e66a88c4404a8a3ac74ccf08f87 2024-11-22T18:58:20,743 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/C/ac7ea58947ff4fc5bf950cf70a840783 is 50, key is test_row_0/C:col10/1732301899507/Put/seqid=0 2024-11-22T18:58:20,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742383_1559 (size=12301) 2024-11-22T18:58:20,815 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=179 2024-11-22T18:58:20,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:20,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. as already flushing 2024-11-22T18:58:20,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:20,816 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] handler.RSProcedureHandler(58): pid=179 java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:20,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=179 java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:20,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=179 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:20,925 INFO [Thread-2189 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2189. 2024-11-22T18:58:20,925 INFO [Thread-2187 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2187. 2024-11-22T18:58:20,925 DEBUG [Thread-2189 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T18:58:20,925 DEBUG [Thread-2187 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T18:58:20,925 DEBUG [Thread-2189 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:58:20,925 DEBUG [Thread-2187 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:58:20,925 DEBUG [Thread-2189 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:58:20,925 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T18:58:20,925 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T18:58:20,926 DEBUG [Thread-2187 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:58:20,926 INFO [Thread-2185 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2185. 2024-11-22T18:58:20,926 DEBUG [Thread-2185 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T18:58:20,926 DEBUG [Thread-2185 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:58:20,926 DEBUG [Thread-2185 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:58:20,926 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T18:58:20,929 INFO [Thread-2183 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2183. 2024-11-22T18:58:20,929 DEBUG [Thread-2183 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T18:58:20,929 DEBUG [Thread-2183 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:58:20,929 INFO [Thread-2191 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2191. 2024-11-22T18:58:20,929 DEBUG [Thread-2191 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T18:58:20,929 DEBUG [Thread-2183 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:58:20,929 DEBUG [Thread-2191 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:58:20,929 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T18:58:20,929 DEBUG [Thread-2191 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:58:20,929 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T18:58:20,968 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=179 2024-11-22T18:58:20,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:20,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. as already flushing 2024-11-22T18:58:20,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:20,969 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] handler.RSProcedureHandler(58): pid=179 java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:20,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=179 java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:20,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=179 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:21,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-11-22T18:58:21,121 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=179 2024-11-22T18:58:21,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:21,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. as already flushing 2024-11-22T18:58:21,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:21,121 ERROR [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] handler.RSProcedureHandler(58): pid=179 java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:21,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=179 java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:21,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4177): Remote procedure failed, pid=179 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T18:58:21,147 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/C/ac7ea58947ff4fc5bf950cf70a840783 2024-11-22T18:58:21,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/A/8bfce39253124ec4af3b9211c3e5b316 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/8bfce39253124ec4af3b9211c3e5b316 2024-11-22T18:58:21,153 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/8bfce39253124ec4af3b9211c3e5b316, entries=200, sequenceid=274, filesize=39.0 K 2024-11-22T18:58:21,154 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/B/9f2b4e66a88c4404a8a3ac74ccf08f87 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/9f2b4e66a88c4404a8a3ac74ccf08f87 2024-11-22T18:58:21,157 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/9f2b4e66a88c4404a8a3ac74ccf08f87, entries=150, sequenceid=274, filesize=12.0 K 2024-11-22T18:58:21,157 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/C/ac7ea58947ff4fc5bf950cf70a840783 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/ac7ea58947ff4fc5bf950cf70a840783 2024-11-22T18:58:21,160 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/ac7ea58947ff4fc5bf950cf70a840783, entries=150, sequenceid=274, filesize=12.0 K 2024-11-22T18:58:21,161 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for cb9c73ed0f14cbdd1f76d53a72aa266b in 1653ms, sequenceid=274, compaction requested=true 2024-11-22T18:58:21,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cb9c73ed0f14cbdd1f76d53a72aa266b: 2024-11-22T18:58:21,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb9c73ed0f14cbdd1f76d53a72aa266b:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T18:58:21,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:58:21,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb9c73ed0f14cbdd1f76d53a72aa266b:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T18:58:21,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:58:21,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cb9c73ed0f14cbdd1f76d53a72aa266b:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T18:58:21,161 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T18:58:21,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:58:21,161 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T18:58:21,162 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142420 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T18:58:21,162 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49266 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T18:58:21,162 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1541): cb9c73ed0f14cbdd1f76d53a72aa266b/A is initiating minor compaction (all files) 2024-11-22T18:58:21,162 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1541): cb9c73ed0f14cbdd1f76d53a72aa266b/B is initiating minor compaction (all files) 2024-11-22T18:58:21,162 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb9c73ed0f14cbdd1f76d53a72aa266b/A in TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:21,162 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb9c73ed0f14cbdd1f76d53a72aa266b/B in TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:21,162 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/567c8d1e8aff4996b0eea1e37dc56698, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/0a28d7a403c54b6c8a6f7beeb856cc53, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/098ee73c21d440feb9ecdd77c759ebfe, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/8bfce39253124ec4af3b9211c3e5b316] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp, totalSize=139.1 K 2024-11-22T18:58:21,162 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/569b558e605c4750a38e0ca52e422350, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/bfac8c50acdf4fd9ad8392ac65566806, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/0754853085a1462bb1d1d5a21b4603ab, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/9f2b4e66a88c4404a8a3ac74ccf08f87] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp, totalSize=48.1 K 2024-11-22T18:58:21,162 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:21,162 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. files: [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/567c8d1e8aff4996b0eea1e37dc56698, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/0a28d7a403c54b6c8a6f7beeb856cc53, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/098ee73c21d440feb9ecdd77c759ebfe, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/8bfce39253124ec4af3b9211c3e5b316] 2024-11-22T18:58:21,163 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 569b558e605c4750a38e0ca52e422350, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732301895718 2024-11-22T18:58:21,163 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 567c8d1e8aff4996b0eea1e37dc56698, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732301895718 2024-11-22T18:58:21,163 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a28d7a403c54b6c8a6f7beeb856cc53, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732301896427 2024-11-22T18:58:21,163 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting bfac8c50acdf4fd9ad8392ac65566806, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732301896444 2024-11-22T18:58:21,163 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 098ee73c21d440feb9ecdd77c759ebfe, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732301897638 2024-11-22T18:58:21,163 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 0754853085a1462bb1d1d5a21b4603ab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732301897638 2024-11-22T18:58:21,163 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8bfce39253124ec4af3b9211c3e5b316, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732301898854 2024-11-22T18:58:21,163 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f2b4e66a88c4404a8a3ac74ccf08f87, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732301898864 2024-11-22T18:58:21,173 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb9c73ed0f14cbdd1f76d53a72aa266b#B#compaction#474 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:58:21,173 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cb9c73ed0f14cbdd1f76d53a72aa266b] 2024-11-22T18:58:21,173 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/B/a5113627838f4ffd9bf0b215666f2b19 is 50, key is test_row_0/B:col10/1732301899507/Put/seqid=0 2024-11-22T18:58:21,174 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411222ae8b68ed44646b6ae61eb3ab7c9a117_cb9c73ed0f14cbdd1f76d53a72aa266b store=[table=TestAcidGuarantees family=A region=cb9c73ed0f14cbdd1f76d53a72aa266b] 2024-11-22T18:58:21,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742384_1560 (size=12949) 2024-11-22T18:58:21,199 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411222ae8b68ed44646b6ae61eb3ab7c9a117_cb9c73ed0f14cbdd1f76d53a72aa266b, store=[table=TestAcidGuarantees family=A region=cb9c73ed0f14cbdd1f76d53a72aa266b] 2024-11-22T18:58:21,199 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411222ae8b68ed44646b6ae61eb3ab7c9a117_cb9c73ed0f14cbdd1f76d53a72aa266b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cb9c73ed0f14cbdd1f76d53a72aa266b] 2024-11-22T18:58:21,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742385_1561 (size=4469) 2024-11-22T18:58:21,273 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43593 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=179 2024-11-22T18:58:21,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:21,274 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegion(2902): Flushing cb9c73ed0f14cbdd1f76d53a72aa266b 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-22T18:58:21,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122ba5989a1647745169dca3433b290b82e_cb9c73ed0f14cbdd1f76d53a72aa266b is 50, key is test_row_0/A:col10/1732301899545/Put/seqid=0 2024-11-22T18:58:21,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742386_1562 (size=12454) 2024-11-22T18:58:21,580 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/B/a5113627838f4ffd9bf0b215666f2b19 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/a5113627838f4ffd9bf0b215666f2b19 2024-11-22T18:58:21,584 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cb9c73ed0f14cbdd1f76d53a72aa266b/B of cb9c73ed0f14cbdd1f76d53a72aa266b into a5113627838f4ffd9bf0b215666f2b19(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:58:21,584 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb9c73ed0f14cbdd1f76d53a72aa266b: 2024-11-22T18:58:21,584 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., storeName=cb9c73ed0f14cbdd1f76d53a72aa266b/B, priority=12, startTime=1732301901161; duration=0sec 2024-11-22T18:58:21,584 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T18:58:21,584 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb9c73ed0f14cbdd1f76d53a72aa266b:B 2024-11-22T18:58:21,584 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T18:58:21,585 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49266 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T18:58:21,585 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1541): cb9c73ed0f14cbdd1f76d53a72aa266b/C is initiating minor compaction (all files) 2024-11-22T18:58:21,585 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cb9c73ed0f14cbdd1f76d53a72aa266b/C in TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:21,585 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/73f7cb21d98042e1b8b28ccb2abff049, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/9d50640af9d9444a9500360b225adaa1, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/7ca0114f5f274ccaa66c25b4997089cd, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/ac7ea58947ff4fc5bf950cf70a840783] into tmpdir=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp, totalSize=48.1 K 2024-11-22T18:58:21,585 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 73f7cb21d98042e1b8b28ccb2abff049, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732301895718 2024-11-22T18:58:21,585 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d50640af9d9444a9500360b225adaa1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732301896444 2024-11-22T18:58:21,586 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ca0114f5f274ccaa66c25b4997089cd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732301897638 2024-11-22T18:58:21,586 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] compactions.Compactor(225): Compacting ac7ea58947ff4fc5bf950cf70a840783, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732301898864 2024-11-22T18:58:21,592 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb9c73ed0f14cbdd1f76d53a72aa266b#C#compaction#477 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:58:21,592 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/C/d243845e4a12486287016e31a90b699c is 50, key is test_row_0/C:col10/1732301899507/Put/seqid=0 2024-11-22T18:58:21,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742387_1563 (size=12949) 2024-11-22T18:58:21,603 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cb9c73ed0f14cbdd1f76d53a72aa266b#A#compaction#475 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T18:58:21,603 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/A/df9e08462b204de08883672235c814af is 175, key is test_row_0/A:col10/1732301899507/Put/seqid=0 2024-11-22T18:58:21,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742388_1564 (size=31903) 2024-11-22T18:58:21,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:21,692 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122ba5989a1647745169dca3433b290b82e_cb9c73ed0f14cbdd1f76d53a72aa266b to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122ba5989a1647745169dca3433b290b82e_cb9c73ed0f14cbdd1f76d53a72aa266b 2024-11-22T18:58:21,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/A/daea90dbaf78476e8745f55c8aea3b44, store: [table=TestAcidGuarantees family=A region=cb9c73ed0f14cbdd1f76d53a72aa266b] 2024-11-22T18:58:21,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/A/daea90dbaf78476e8745f55c8aea3b44 is 175, key is test_row_0/A:col10/1732301899545/Put/seqid=0 2024-11-22T18:58:21,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742389_1565 (size=31255) 2024-11-22T18:58:21,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43593 {}] regionserver.HRegion(8855): Flush requested on cb9c73ed0f14cbdd1f76d53a72aa266b 2024-11-22T18:58:21,727 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. as already flushing 2024-11-22T18:58:21,727 INFO [Thread-2172 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2172. 2024-11-22T18:58:21,727 DEBUG [Thread-2172 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T18:58:21,727 DEBUG [Thread-2172 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:58:21,727 DEBUG [Thread-2172 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:58:21,727 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T18:58:21,999 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/C/d243845e4a12486287016e31a90b699c as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/d243845e4a12486287016e31a90b699c 2024-11-22T18:58:22,002 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cb9c73ed0f14cbdd1f76d53a72aa266b/C of cb9c73ed0f14cbdd1f76d53a72aa266b into d243845e4a12486287016e31a90b699c(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:58:22,002 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb9c73ed0f14cbdd1f76d53a72aa266b: 2024-11-22T18:58:22,002 INFO [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., storeName=cb9c73ed0f14cbdd1f76d53a72aa266b/C, priority=12, startTime=1732301901161; duration=0sec 2024-11-22T18:58:22,002 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:58:22,002 DEBUG [RS:0;d79ba0c344fb:43593-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb9c73ed0f14cbdd1f76d53a72aa266b:C 2024-11-22T18:58:22,009 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/A/df9e08462b204de08883672235c814af as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/df9e08462b204de08883672235c814af 2024-11-22T18:58:22,012 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cb9c73ed0f14cbdd1f76d53a72aa266b/A of cb9c73ed0f14cbdd1f76d53a72aa266b into df9e08462b204de08883672235c814af(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T18:58:22,012 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cb9c73ed0f14cbdd1f76d53a72aa266b: 2024-11-22T18:58:22,012 INFO [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., storeName=cb9c73ed0f14cbdd1f76d53a72aa266b/A, priority=12, startTime=1732301901161; duration=0sec 2024-11-22T18:58:22,013 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T18:58:22,013 DEBUG [RS:0;d79ba0c344fb:43593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cb9c73ed0f14cbdd1f76d53a72aa266b:A 2024-11-22T18:58:22,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-11-22T18:58:22,097 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=289, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/A/daea90dbaf78476e8745f55c8aea3b44 2024-11-22T18:58:22,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/B/f06c3cc793ab4be19b5f493a96a69d9c is 50, key is test_row_0/B:col10/1732301899545/Put/seqid=0 2024-11-22T18:58:22,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742390_1566 (size=12301) 2024-11-22T18:58:22,429 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-22T18:58:22,506 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/B/f06c3cc793ab4be19b5f493a96a69d9c 2024-11-22T18:58:22,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/C/8bde12bc7bcf48d495a0017a8e74d949 is 50, key is test_row_0/C:col10/1732301899545/Put/seqid=0 2024-11-22T18:58:22,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742391_1567 (size=12301) 2024-11-22T18:58:22,916 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/C/8bde12bc7bcf48d495a0017a8e74d949 2024-11-22T18:58:22,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/A/daea90dbaf78476e8745f55c8aea3b44 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/daea90dbaf78476e8745f55c8aea3b44 2024-11-22T18:58:22,922 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/daea90dbaf78476e8745f55c8aea3b44, entries=150, sequenceid=289, filesize=30.5 K 2024-11-22T18:58:22,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/B/f06c3cc793ab4be19b5f493a96a69d9c as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/f06c3cc793ab4be19b5f493a96a69d9c 2024-11-22T18:58:22,925 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/f06c3cc793ab4be19b5f493a96a69d9c, entries=150, sequenceid=289, filesize=12.0 K 2024-11-22T18:58:22,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/C/8bde12bc7bcf48d495a0017a8e74d949 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/8bde12bc7bcf48d495a0017a8e74d949 2024-11-22T18:58:22,928 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/8bde12bc7bcf48d495a0017a8e74d949, entries=150, sequenceid=289, filesize=12.0 K 2024-11-22T18:58:22,928 INFO [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=6.71 KB/6870 for cb9c73ed0f14cbdd1f76d53a72aa266b in 1654ms, sequenceid=289, compaction requested=false 2024-11-22T18:58:22,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegion(2603): Flush status journal for cb9c73ed0f14cbdd1f76d53a72aa266b: 2024-11-22T18:58:22,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:22,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d79ba0c344fb:0-1 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=179 2024-11-22T18:58:22,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster(4169): Remote procedure done, pid=179 2024-11-22T18:58:22,931 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=179, resume processing ppid=178 2024-11-22T18:58:22,931 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=179, ppid=178, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0310 sec 2024-11-22T18:58:22,932 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=178, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=178, table=TestAcidGuarantees in 3.0350 sec 2024-11-22T18:58:24,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-11-22T18:58:24,066 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T18:58:29,587 INFO [Thread-2178 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2178. 2024-11-22T18:58:29,587 DEBUG [Thread-2178 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T18:58:29,587 DEBUG [Thread-2178 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:58:29,587 DEBUG [Thread-2178 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:58:29,587 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T18:58:29,607 INFO [Thread-2174 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2174. 2024-11-22T18:58:29,607 DEBUG [Thread-2174 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T18:58:29,607 DEBUG [Thread-2174 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:58:29,607 DEBUG [Thread-2174 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:58:29,607 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T18:58:29,697 INFO [Thread-2180 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2180. 2024-11-22T18:58:29,697 INFO [Thread-2176 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2176. 2024-11-22T18:58:29,697 DEBUG [Thread-2180 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T18:58:29,697 DEBUG [Thread-2180 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:58:29,697 DEBUG [Thread-2176 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T18:58:29,697 DEBUG [Thread-2176 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:58:29,697 DEBUG [Thread-2180 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:58:29,697 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T18:58:29,697 DEBUG [Thread-2176 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:58:29,697 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T18:58:29,697 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-22T18:58:29,697 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 191 2024-11-22T18:58:29,698 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 9 2024-11-22T18:58:29,698 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 10 2024-11-22T18:58:29,698 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 11 2024-11-22T18:58:29,698 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 9 2024-11-22T18:58:29,698 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-22T18:58:29,698 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3927 2024-11-22T18:58:29,698 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3763 2024-11-22T18:58:29,698 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3507 2024-11-22T18:58:29,698 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3927 2024-11-22T18:58:29,698 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3749 2024-11-22T18:58:29,698 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-22T18:58:29,698 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-22T18:58:29,698 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T18:58:29,698 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:58:29,698 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:58:29,698 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T18:58:29,699 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-22T18:58:29,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] procedure2.ProcedureExecutor(1139): Stored pid=180, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-22T18:58:29,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=180 2024-11-22T18:58:29,702 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732301909701"}]},"ts":"1732301909701"} 2024-11-22T18:58:29,703 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-22T18:58:29,703 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-22T18:58:29,703 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=181, ppid=180, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-22T18:58:29,704 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=182, ppid=181, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cb9c73ed0f14cbdd1f76d53a72aa266b, UNASSIGN}] 2024-11-22T18:58:29,705 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=182, ppid=181, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cb9c73ed0f14cbdd1f76d53a72aa266b, UNASSIGN 2024-11-22T18:58:29,706 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=182 updating hbase:meta row=cb9c73ed0f14cbdd1f76d53a72aa266b, regionState=CLOSING, regionLocation=d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:29,707 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=182, ppid=181, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cb9c73ed0f14cbdd1f76d53a72aa266b, UNASSIGN because future has completed 2024-11-22T18:58:29,707 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-22T18:58:29,707 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=183, ppid=182, state=RUNNABLE, hasLock=false; CloseRegionProcedure cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173}] 2024-11-22T18:58:29,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=180 2024-11-22T18:58:29,859 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] handler.UnassignRegionHandler(122): Close cb9c73ed0f14cbdd1f76d53a72aa266b 2024-11-22T18:58:29,859 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-22T18:58:29,859 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] regionserver.HRegion(1722): Closing cb9c73ed0f14cbdd1f76d53a72aa266b, disabling compactions & flushes 2024-11-22T18:58:29,859 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:29,859 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:29,859 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. after waiting 0 ms 2024-11-22T18:58:29,859 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:29,859 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] regionserver.HRegion(2902): Flushing cb9c73ed0f14cbdd1f76d53a72aa266b 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-22T18:58:29,864 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122e6b0b7a5bc574d4bbf71a884814592ab_cb9c73ed0f14cbdd1f76d53a72aa266b is 50, key is test_row_0/A:col10/1732301909606/Put/seqid=0 2024-11-22T18:58:29,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742392_1568 (size=12454) 2024-11-22T18:58:30,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=180 2024-11-22T18:58:30,268 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T18:58:30,271 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122e6b0b7a5bc574d4bbf71a884814592ab_cb9c73ed0f14cbdd1f76d53a72aa266b to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122e6b0b7a5bc574d4bbf71a884814592ab_cb9c73ed0f14cbdd1f76d53a72aa266b 2024-11-22T18:58:30,272 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/A/1b0682d926b34807aef17434cd3dbd03, store: [table=TestAcidGuarantees family=A region=cb9c73ed0f14cbdd1f76d53a72aa266b] 2024-11-22T18:58:30,272 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/A/1b0682d926b34807aef17434cd3dbd03 is 175, key is test_row_0/A:col10/1732301909606/Put/seqid=0 2024-11-22T18:58:30,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742393_1569 (size=31255) 2024-11-22T18:58:30,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=180 2024-11-22T18:58:30,676 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=300, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/A/1b0682d926b34807aef17434cd3dbd03 2024-11-22T18:58:30,681 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/B/06e98fe6e9e84b4589ebfb3d5eab404c is 50, key is test_row_0/B:col10/1732301909606/Put/seqid=0 2024-11-22T18:58:30,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742394_1570 (size=12301) 2024-11-22T18:58:30,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=180 2024-11-22T18:58:31,085 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/B/06e98fe6e9e84b4589ebfb3d5eab404c 2024-11-22T18:58:31,090 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/C/47f30ae22e5d4e5998c0e5e215c84bb1 is 50, key is test_row_0/C:col10/1732301909606/Put/seqid=0 2024-11-22T18:58:31,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742395_1571 (size=12301) 2024-11-22T18:58:31,494 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/C/47f30ae22e5d4e5998c0e5e215c84bb1 2024-11-22T18:58:31,497 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/A/1b0682d926b34807aef17434cd3dbd03 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/1b0682d926b34807aef17434cd3dbd03 2024-11-22T18:58:31,500 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/1b0682d926b34807aef17434cd3dbd03, entries=150, sequenceid=300, filesize=30.5 K 2024-11-22T18:58:31,500 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/B/06e98fe6e9e84b4589ebfb3d5eab404c as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/06e98fe6e9e84b4589ebfb3d5eab404c 2024-11-22T18:58:31,503 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/06e98fe6e9e84b4589ebfb3d5eab404c, entries=150, sequenceid=300, filesize=12.0 K 2024-11-22T18:58:31,503 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/.tmp/C/47f30ae22e5d4e5998c0e5e215c84bb1 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/47f30ae22e5d4e5998c0e5e215c84bb1 2024-11-22T18:58:31,506 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/47f30ae22e5d4e5998c0e5e215c84bb1, entries=150, sequenceid=300, filesize=12.0 K 2024-11-22T18:58:31,506 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for cb9c73ed0f14cbdd1f76d53a72aa266b in 1647ms, sequenceid=300, compaction requested=true 2024-11-22T18:58:31,507 DEBUG [StoreCloser-TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/e679fead15da4ede900c00a6dcbd7692, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/d49ace551f5542b8b4c5bfab9e7d7503, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/80f4a74df7a740548a92bc02374ec449, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/965e04ca1e6a4abe8dd2e06e77e998b6, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/531cc4f34d6448d09db4f2ac552b1122, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/45637cefa2cf47e0b946c26182931f17, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/3423d2c747a04471aed42b9eaf138953, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/af354a9ad25547f1836f382459969358, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/33992c38c538448fbd9949ac6d6161d2, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/bebc5c22d2624e008aa54a68df6c3b6c, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/57989b4436df4983a952e0766dd7db07, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/a911f63f17f04ff78c63a4b77064477b, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/e1a88c3b310f420b8af0eac7547d1190, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/1d91fcb19abd4069a0cad811bddc4ce1, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/916c2bddf8594497a82d9e5408dc02e3, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/567c8d1e8aff4996b0eea1e37dc56698, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/0a28d7a403c54b6c8a6f7beeb856cc53, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/098ee73c21d440feb9ecdd77c759ebfe, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/8bfce39253124ec4af3b9211c3e5b316] to archive 2024-11-22T18:58:31,507 DEBUG [StoreCloser-TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T18:58:31,509 DEBUG [StoreCloser-TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/e679fead15da4ede900c00a6dcbd7692 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/e679fead15da4ede900c00a6dcbd7692 2024-11-22T18:58:31,510 DEBUG [StoreCloser-TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/d49ace551f5542b8b4c5bfab9e7d7503 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/d49ace551f5542b8b4c5bfab9e7d7503 2024-11-22T18:58:31,511 DEBUG [StoreCloser-TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/80f4a74df7a740548a92bc02374ec449 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/80f4a74df7a740548a92bc02374ec449 2024-11-22T18:58:31,511 DEBUG [StoreCloser-TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/965e04ca1e6a4abe8dd2e06e77e998b6 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/965e04ca1e6a4abe8dd2e06e77e998b6 2024-11-22T18:58:31,512 DEBUG [StoreCloser-TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/531cc4f34d6448d09db4f2ac552b1122 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/531cc4f34d6448d09db4f2ac552b1122 2024-11-22T18:58:31,513 DEBUG [StoreCloser-TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/45637cefa2cf47e0b946c26182931f17 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/45637cefa2cf47e0b946c26182931f17 2024-11-22T18:58:31,514 DEBUG [StoreCloser-TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/3423d2c747a04471aed42b9eaf138953 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/3423d2c747a04471aed42b9eaf138953 2024-11-22T18:58:31,515 DEBUG [StoreCloser-TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/af354a9ad25547f1836f382459969358 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/af354a9ad25547f1836f382459969358 2024-11-22T18:58:31,516 DEBUG [StoreCloser-TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/33992c38c538448fbd9949ac6d6161d2 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/33992c38c538448fbd9949ac6d6161d2 2024-11-22T18:58:31,517 DEBUG [StoreCloser-TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/bebc5c22d2624e008aa54a68df6c3b6c to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/bebc5c22d2624e008aa54a68df6c3b6c 2024-11-22T18:58:31,517 DEBUG [StoreCloser-TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/57989b4436df4983a952e0766dd7db07 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/57989b4436df4983a952e0766dd7db07 2024-11-22T18:58:31,518 DEBUG [StoreCloser-TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/a911f63f17f04ff78c63a4b77064477b to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/a911f63f17f04ff78c63a4b77064477b 2024-11-22T18:58:31,519 DEBUG [StoreCloser-TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/e1a88c3b310f420b8af0eac7547d1190 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/e1a88c3b310f420b8af0eac7547d1190 2024-11-22T18:58:31,519 DEBUG [StoreCloser-TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/1d91fcb19abd4069a0cad811bddc4ce1 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/1d91fcb19abd4069a0cad811bddc4ce1 2024-11-22T18:58:31,520 DEBUG [StoreCloser-TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/916c2bddf8594497a82d9e5408dc02e3 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/916c2bddf8594497a82d9e5408dc02e3 2024-11-22T18:58:31,521 DEBUG [StoreCloser-TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/567c8d1e8aff4996b0eea1e37dc56698 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/567c8d1e8aff4996b0eea1e37dc56698 2024-11-22T18:58:31,522 DEBUG [StoreCloser-TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/0a28d7a403c54b6c8a6f7beeb856cc53 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/0a28d7a403c54b6c8a6f7beeb856cc53 2024-11-22T18:58:31,522 DEBUG [StoreCloser-TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/098ee73c21d440feb9ecdd77c759ebfe to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/098ee73c21d440feb9ecdd77c759ebfe 2024-11-22T18:58:31,523 DEBUG [StoreCloser-TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/8bfce39253124ec4af3b9211c3e5b316 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/8bfce39253124ec4af3b9211c3e5b316 2024-11-22T18:58:31,524 DEBUG [StoreCloser-TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/f0902f4602d64af09204bbd2487010a1, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/2666f98fcf074559a49389cd6ede0d04, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/da2146315f5540b9b7bb2d887b04e6cf, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/2fdf3ea17fe94233b5dae5adb696947d, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/b01171d576ef47209a5d1e973b764bc9, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/a405d3ee57604c389a655cede2c646d3, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/9b821b53abfc4b2aac21f4017f392d80, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/5ffe7da2c2ff44b782672cb1978646c0, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/c89bfab893214769bb7e92bf7643e4e6, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/c11e007e2220474bb8fcffc10636145d, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/b91cbf6785b44fdb88b56b45e8e5b8e7, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/92c73f41bd6a49d79c50fd4b888e1129, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/7de76d9f7c374e51b983abfc78db2ed5, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/5685e6bb4b9741f4b843365e91e80467, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/569b558e605c4750a38e0ca52e422350, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/07dfcd11decf4500843e7e96919c9f6b, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/bfac8c50acdf4fd9ad8392ac65566806, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/0754853085a1462bb1d1d5a21b4603ab, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/9f2b4e66a88c4404a8a3ac74ccf08f87] to archive 2024-11-22T18:58:31,525 DEBUG [StoreCloser-TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T18:58:31,526 DEBUG [StoreCloser-TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/f0902f4602d64af09204bbd2487010a1 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/f0902f4602d64af09204bbd2487010a1 2024-11-22T18:58:31,526 DEBUG [StoreCloser-TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/2666f98fcf074559a49389cd6ede0d04 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/2666f98fcf074559a49389cd6ede0d04 2024-11-22T18:58:31,527 DEBUG [StoreCloser-TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/da2146315f5540b9b7bb2d887b04e6cf to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/da2146315f5540b9b7bb2d887b04e6cf 2024-11-22T18:58:31,528 DEBUG [StoreCloser-TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/2fdf3ea17fe94233b5dae5adb696947d to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/2fdf3ea17fe94233b5dae5adb696947d 2024-11-22T18:58:31,529 DEBUG [StoreCloser-TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/b01171d576ef47209a5d1e973b764bc9 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/b01171d576ef47209a5d1e973b764bc9 2024-11-22T18:58:31,529 DEBUG [StoreCloser-TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/a405d3ee57604c389a655cede2c646d3 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/a405d3ee57604c389a655cede2c646d3 2024-11-22T18:58:31,530 DEBUG [StoreCloser-TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/9b821b53abfc4b2aac21f4017f392d80 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/9b821b53abfc4b2aac21f4017f392d80 2024-11-22T18:58:31,531 DEBUG [StoreCloser-TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/5ffe7da2c2ff44b782672cb1978646c0 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/5ffe7da2c2ff44b782672cb1978646c0 2024-11-22T18:58:31,532 DEBUG [StoreCloser-TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/c89bfab893214769bb7e92bf7643e4e6 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/c89bfab893214769bb7e92bf7643e4e6 2024-11-22T18:58:31,532 DEBUG [StoreCloser-TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/c11e007e2220474bb8fcffc10636145d to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/c11e007e2220474bb8fcffc10636145d 2024-11-22T18:58:31,533 DEBUG [StoreCloser-TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/b91cbf6785b44fdb88b56b45e8e5b8e7 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/b91cbf6785b44fdb88b56b45e8e5b8e7 2024-11-22T18:58:31,534 DEBUG [StoreCloser-TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/92c73f41bd6a49d79c50fd4b888e1129 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/92c73f41bd6a49d79c50fd4b888e1129 2024-11-22T18:58:31,534 DEBUG [StoreCloser-TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/7de76d9f7c374e51b983abfc78db2ed5 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/7de76d9f7c374e51b983abfc78db2ed5 2024-11-22T18:58:31,535 DEBUG [StoreCloser-TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/5685e6bb4b9741f4b843365e91e80467 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/5685e6bb4b9741f4b843365e91e80467 2024-11-22T18:58:31,536 DEBUG [StoreCloser-TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/569b558e605c4750a38e0ca52e422350 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/569b558e605c4750a38e0ca52e422350 2024-11-22T18:58:31,537 DEBUG [StoreCloser-TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/07dfcd11decf4500843e7e96919c9f6b to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/07dfcd11decf4500843e7e96919c9f6b 2024-11-22T18:58:31,537 DEBUG [StoreCloser-TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/bfac8c50acdf4fd9ad8392ac65566806 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/bfac8c50acdf4fd9ad8392ac65566806 2024-11-22T18:58:31,538 DEBUG [StoreCloser-TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/0754853085a1462bb1d1d5a21b4603ab to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/0754853085a1462bb1d1d5a21b4603ab 2024-11-22T18:58:31,539 DEBUG [StoreCloser-TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/9f2b4e66a88c4404a8a3ac74ccf08f87 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/9f2b4e66a88c4404a8a3ac74ccf08f87 2024-11-22T18:58:31,540 DEBUG [StoreCloser-TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/02b9a466422e4ff98857ec3fc2dc9b78, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/2d3b222740b74d59830ddc1efd70ed96, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/1f89af3a5406457aa351ec87a0d7fe65, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/cc818d22fb644cbd8872880e5a30c64f, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/d9765992869f4e06bc51ec1401702ddc, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/79e4d06e990f42a1b6dea756723d5114, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/dd84de494dc246f3846e9a5c3bc9b6ed, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/1c8b70a033b64d25b16884b15ce13417, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/8d548bcc855646d9b3af0a8cb7efc454, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/93656e82028f4024b64d966f570358da, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/f5e049c735214bbc84d25627ce62fe2c, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/1e3e0909dcbb4986b26a2dc52f5348fe, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/0ff0d34b533f44b283a52db06472309d, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/6c267aee58404e259b2077c8169b0344, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/73f7cb21d98042e1b8b28ccb2abff049, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/216ec2b48ed3447e954ee0ccb5aa44f6, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/9d50640af9d9444a9500360b225adaa1, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/7ca0114f5f274ccaa66c25b4997089cd, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/ac7ea58947ff4fc5bf950cf70a840783] to archive 2024-11-22T18:58:31,541 DEBUG [StoreCloser-TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T18:58:31,542 DEBUG [StoreCloser-TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/02b9a466422e4ff98857ec3fc2dc9b78 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/02b9a466422e4ff98857ec3fc2dc9b78 2024-11-22T18:58:31,543 DEBUG [StoreCloser-TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/2d3b222740b74d59830ddc1efd70ed96 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/2d3b222740b74d59830ddc1efd70ed96 2024-11-22T18:58:31,544 DEBUG [StoreCloser-TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/1f89af3a5406457aa351ec87a0d7fe65 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/1f89af3a5406457aa351ec87a0d7fe65 2024-11-22T18:58:31,545 DEBUG [StoreCloser-TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/cc818d22fb644cbd8872880e5a30c64f to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/cc818d22fb644cbd8872880e5a30c64f 2024-11-22T18:58:31,546 DEBUG [StoreCloser-TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/d9765992869f4e06bc51ec1401702ddc to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/d9765992869f4e06bc51ec1401702ddc 2024-11-22T18:58:31,547 DEBUG [StoreCloser-TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/79e4d06e990f42a1b6dea756723d5114 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/79e4d06e990f42a1b6dea756723d5114 2024-11-22T18:58:31,548 DEBUG [StoreCloser-TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/dd84de494dc246f3846e9a5c3bc9b6ed to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/dd84de494dc246f3846e9a5c3bc9b6ed 2024-11-22T18:58:31,548 DEBUG [StoreCloser-TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/1c8b70a033b64d25b16884b15ce13417 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/1c8b70a033b64d25b16884b15ce13417 2024-11-22T18:58:31,549 DEBUG [StoreCloser-TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/8d548bcc855646d9b3af0a8cb7efc454 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/8d548bcc855646d9b3af0a8cb7efc454 2024-11-22T18:58:31,550 DEBUG [StoreCloser-TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/93656e82028f4024b64d966f570358da to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/93656e82028f4024b64d966f570358da 2024-11-22T18:58:31,551 DEBUG [StoreCloser-TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/f5e049c735214bbc84d25627ce62fe2c to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/f5e049c735214bbc84d25627ce62fe2c 2024-11-22T18:58:31,552 DEBUG [StoreCloser-TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/1e3e0909dcbb4986b26a2dc52f5348fe to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/1e3e0909dcbb4986b26a2dc52f5348fe 2024-11-22T18:58:31,552 DEBUG [StoreCloser-TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/0ff0d34b533f44b283a52db06472309d to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/0ff0d34b533f44b283a52db06472309d 2024-11-22T18:58:31,553 DEBUG [StoreCloser-TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/6c267aee58404e259b2077c8169b0344 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/6c267aee58404e259b2077c8169b0344 2024-11-22T18:58:31,554 DEBUG [StoreCloser-TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/73f7cb21d98042e1b8b28ccb2abff049 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/73f7cb21d98042e1b8b28ccb2abff049 2024-11-22T18:58:31,555 DEBUG [StoreCloser-TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/216ec2b48ed3447e954ee0ccb5aa44f6 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/216ec2b48ed3447e954ee0ccb5aa44f6 2024-11-22T18:58:31,556 DEBUG [StoreCloser-TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/9d50640af9d9444a9500360b225adaa1 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/9d50640af9d9444a9500360b225adaa1 2024-11-22T18:58:31,556 DEBUG [StoreCloser-TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/7ca0114f5f274ccaa66c25b4997089cd to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/7ca0114f5f274ccaa66c25b4997089cd 2024-11-22T18:58:31,558 DEBUG [StoreCloser-TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/ac7ea58947ff4fc5bf950cf70a840783 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/ac7ea58947ff4fc5bf950cf70a840783 2024-11-22T18:58:31,562 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/recovered.edits/303.seqid, newMaxSeqId=303, maxSeqId=4 2024-11-22T18:58:31,562 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. 2024-11-22T18:58:31,563 DEBUG [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] regionserver.HRegion(1676): Region close journal for cb9c73ed0f14cbdd1f76d53a72aa266b: Waiting for close lock at 1732301909859Running coprocessor pre-close hooks at 1732301909859Disabling compacts and flushes for region at 1732301909859Disabling writes for close at 1732301909859Obtaining lock to block concurrent updates at 1732301909859Preparing flush snapshotting stores in cb9c73ed0f14cbdd1f76d53a72aa266b at 1732301909859Finished memstore snapshotting TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1732301909860 (+1 ms)Flushing stores of TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b. at 1732301909860Flushing cb9c73ed0f14cbdd1f76d53a72aa266b/A: creating writer at 1732301909860Flushing mob file cb9c73ed0f14cbdd1f76d53a72aa266b/A: appending metadata at 1732301909864 (+4 ms)Flushing mob file cb9c73ed0f14cbdd1f76d53a72aa266b/A: closing flushed file at 1732301909864Flushing cb9c73ed0f14cbdd1f76d53a72aa266b/A: appending metadata at 1732301910272 (+408 ms)Flushing cb9c73ed0f14cbdd1f76d53a72aa266b/A: closing flushed file at 1732301910272Flushing cb9c73ed0f14cbdd1f76d53a72aa266b/B: creating writer at 1732301910679 (+407 ms)Flushing cb9c73ed0f14cbdd1f76d53a72aa266b/B: appending metadata at 1732301910681 (+2 ms)Flushing cb9c73ed0f14cbdd1f76d53a72aa266b/B: closing flushed file at 1732301910681Flushing cb9c73ed0f14cbdd1f76d53a72aa266b/C: creating writer at 1732301911088 (+407 ms)Flushing cb9c73ed0f14cbdd1f76d53a72aa266b/C: appending metadata at 1732301911090 (+2 ms)Flushing cb9c73ed0f14cbdd1f76d53a72aa266b/C: closing flushed file at 1732301911090Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@30fc7ecf: reopening flushed file at 1732301911497 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@62de4243: reopening flushed file at 1732301911500 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@13f642b6: reopening flushed file at 1732301911503 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for cb9c73ed0f14cbdd1f76d53a72aa266b in 1647ms, sequenceid=300, compaction requested=true at 1732301911506 (+3 ms)Writing region close event to WAL at 1732301911559 (+53 ms)Running coprocessor post-close hooks at 1732301911562 (+3 ms)Closed at 1732301911562 2024-11-22T18:58:31,564 INFO [RS_CLOSE_REGION-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=183}] handler.UnassignRegionHandler(157): Closed cb9c73ed0f14cbdd1f76d53a72aa266b 2024-11-22T18:58:31,565 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=182 updating hbase:meta row=cb9c73ed0f14cbdd1f76d53a72aa266b, regionState=CLOSED 2024-11-22T18:58:31,566 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=183, ppid=182, state=RUNNABLE, hasLock=false; CloseRegionProcedure cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 because future has completed 2024-11-22T18:58:31,568 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=183, resume processing ppid=182 2024-11-22T18:58:31,568 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=183, ppid=182, state=SUCCESS, hasLock=false; CloseRegionProcedure cb9c73ed0f14cbdd1f76d53a72aa266b, server=d79ba0c344fb,43593,1732301724173 in 1.8600 sec 2024-11-22T18:58:31,569 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=182, resume processing ppid=181 2024-11-22T18:58:31,569 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=182, ppid=181, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cb9c73ed0f14cbdd1f76d53a72aa266b, UNASSIGN in 1.8640 sec 2024-11-22T18:58:31,571 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=181, resume processing ppid=180 2024-11-22T18:58:31,571 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=181, ppid=180, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8670 sec 2024-11-22T18:58:31,572 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732301911572"}]},"ts":"1732301911572"} 2024-11-22T18:58:31,573 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-22T18:58:31,573 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-22T18:58:31,575 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=180, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8740 sec 2024-11-22T18:58:31,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=180 2024-11-22T18:58:31,846 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-22T18:58:31,847 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-22T18:58:31,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] procedure2.ProcedureExecutor(1139): Stored pid=184, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T18:58:31,848 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=184, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T18:58:31,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=184 2024-11-22T18:58:31,848 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=184, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T18:58:31,850 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b 2024-11-22T18:58:31,852 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A, FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B, FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C, FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/recovered.edits] 2024-11-22T18:58:31,854 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/1b0682d926b34807aef17434cd3dbd03 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/1b0682d926b34807aef17434cd3dbd03 2024-11-22T18:58:31,855 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/daea90dbaf78476e8745f55c8aea3b44 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/daea90dbaf78476e8745f55c8aea3b44 2024-11-22T18:58:31,855 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/df9e08462b204de08883672235c814af to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/A/df9e08462b204de08883672235c814af 2024-11-22T18:58:31,857 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/06e98fe6e9e84b4589ebfb3d5eab404c to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/06e98fe6e9e84b4589ebfb3d5eab404c 2024-11-22T18:58:31,858 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/a5113627838f4ffd9bf0b215666f2b19 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/a5113627838f4ffd9bf0b215666f2b19 2024-11-22T18:58:31,858 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/f06c3cc793ab4be19b5f493a96a69d9c to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/B/f06c3cc793ab4be19b5f493a96a69d9c 2024-11-22T18:58:31,860 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/47f30ae22e5d4e5998c0e5e215c84bb1 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/47f30ae22e5d4e5998c0e5e215c84bb1 2024-11-22T18:58:31,861 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/8bde12bc7bcf48d495a0017a8e74d949 to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/8bde12bc7bcf48d495a0017a8e74d949 2024-11-22T18:58:31,861 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/d243845e4a12486287016e31a90b699c to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/C/d243845e4a12486287016e31a90b699c 2024-11-22T18:58:31,863 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/recovered.edits/303.seqid to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b/recovered.edits/303.seqid 2024-11-22T18:58:31,863 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/default/TestAcidGuarantees/cb9c73ed0f14cbdd1f76d53a72aa266b 2024-11-22T18:58:31,863 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-22T18:58:31,864 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-22T18:58:31,864 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-22T18:58:31,866 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122188cebeaad794cc295d2666ab042407e_cb9c73ed0f14cbdd1f76d53a72aa266b to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122188cebeaad794cc295d2666ab042407e_cb9c73ed0f14cbdd1f76d53a72aa266b 2024-11-22T18:58:31,867 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411224cdd0d6c350b40f78dd25f49d0edb927_cb9c73ed0f14cbdd1f76d53a72aa266b to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411224cdd0d6c350b40f78dd25f49d0edb927_cb9c73ed0f14cbdd1f76d53a72aa266b 2024-11-22T18:58:31,867 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112255b3b44f45e548478f031f33bb3c5374_cb9c73ed0f14cbdd1f76d53a72aa266b to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112255b3b44f45e548478f031f33bb3c5374_cb9c73ed0f14cbdd1f76d53a72aa266b 2024-11-22T18:58:31,868 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122702198abbfe34a3ca02d2f6a517c2164_cb9c73ed0f14cbdd1f76d53a72aa266b to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122702198abbfe34a3ca02d2f6a517c2164_cb9c73ed0f14cbdd1f76d53a72aa266b 2024-11-22T18:58:31,869 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122731106beece44fd1a59fcbc66be65b71_cb9c73ed0f14cbdd1f76d53a72aa266b to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122731106beece44fd1a59fcbc66be65b71_cb9c73ed0f14cbdd1f76d53a72aa266b 2024-11-22T18:58:31,871 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112273fd5e1691f84bd6993550be7cd5225b_cb9c73ed0f14cbdd1f76d53a72aa266b to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112273fd5e1691f84bd6993550be7cd5225b_cb9c73ed0f14cbdd1f76d53a72aa266b 2024-11-22T18:58:31,872 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122b7db3e0f83984124bb647f005405862b_cb9c73ed0f14cbdd1f76d53a72aa266b to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122b7db3e0f83984124bb647f005405862b_cb9c73ed0f14cbdd1f76d53a72aa266b 2024-11-22T18:58:31,873 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122b8853b4d56d448b7a29c7d1c09e9ba0c_cb9c73ed0f14cbdd1f76d53a72aa266b to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122b8853b4d56d448b7a29c7d1c09e9ba0c_cb9c73ed0f14cbdd1f76d53a72aa266b 2024-11-22T18:58:31,874 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122ba5989a1647745169dca3433b290b82e_cb9c73ed0f14cbdd1f76d53a72aa266b to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122ba5989a1647745169dca3433b290b82e_cb9c73ed0f14cbdd1f76d53a72aa266b 2024-11-22T18:58:31,874 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122bee51b8f8922410aa21933862d4a9da0_cb9c73ed0f14cbdd1f76d53a72aa266b to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122bee51b8f8922410aa21933862d4a9da0_cb9c73ed0f14cbdd1f76d53a72aa266b 2024-11-22T18:58:31,875 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122d2b00939cd514c50ad1425417dfda49f_cb9c73ed0f14cbdd1f76d53a72aa266b to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122d2b00939cd514c50ad1425417dfda49f_cb9c73ed0f14cbdd1f76d53a72aa266b 2024-11-22T18:58:31,876 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122e6b0b7a5bc574d4bbf71a884814592ab_cb9c73ed0f14cbdd1f76d53a72aa266b to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122e6b0b7a5bc574d4bbf71a884814592ab_cb9c73ed0f14cbdd1f76d53a72aa266b 2024-11-22T18:58:31,877 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122e76bed5978624b57927cccbba6be55df_cb9c73ed0f14cbdd1f76d53a72aa266b to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122e76bed5978624b57927cccbba6be55df_cb9c73ed0f14cbdd1f76d53a72aa266b 2024-11-22T18:58:31,878 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122e934d832415e4cababf9211a65401528_cb9c73ed0f14cbdd1f76d53a72aa266b to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122e934d832415e4cababf9211a65401528_cb9c73ed0f14cbdd1f76d53a72aa266b 2024-11-22T18:58:31,878 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122f14f02db921a4e9588dace55fbc56e1c_cb9c73ed0f14cbdd1f76d53a72aa266b to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122f14f02db921a4e9588dace55fbc56e1c_cb9c73ed0f14cbdd1f76d53a72aa266b 2024-11-22T18:58:31,879 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122f9b600011b324dbcbf1eff85a4a15104_cb9c73ed0f14cbdd1f76d53a72aa266b to hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122f9b600011b324dbcbf1eff85a4a15104_cb9c73ed0f14cbdd1f76d53a72aa266b 2024-11-22T18:58:31,879 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-22T18:58:31,881 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=184, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T18:58:31,883 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-22T18:58:31,885 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-22T18:58:31,886 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=184, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T18:58:31,886 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-22T18:58:31,886 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732301911886"}]},"ts":"9223372036854775807"} 2024-11-22T18:58:31,888 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-22T18:58:31,888 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => cb9c73ed0f14cbdd1f76d53a72aa266b, NAME => 'TestAcidGuarantees,,1732301877817.cb9c73ed0f14cbdd1f76d53a72aa266b.', STARTKEY => '', ENDKEY => ''}] 2024-11-22T18:58:31,888 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-22T18:58:31,888 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732301911888"}]},"ts":"9223372036854775807"} 2024-11-22T18:58:31,890 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-22T18:58:31,890 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=184, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T18:58:31,891 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=184, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 43 msec 2024-11-22T18:58:31,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=184 2024-11-22T18:58:31,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-22T18:58:31,956 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-22T18:58:31,966 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=237 (was 238), OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=346 (was 301) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7365 (was 7840) 2024-11-22T18:58:31,966 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-22T18:58:31,966 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-22T18:58:31,966 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T18:58:31,966 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:58:31,966 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:58:31,966 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T18:58:31,966 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-22T18:58:31,966 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1107072306, stopped=false 2024-11-22T18:58:31,967 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=d79ba0c344fb,44109,1732301723398 2024-11-22T18:58:31,969 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44109-0x10141087e180000, quorum=127.0.0.1:62869, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-22T18:58:31,969 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43593-0x10141087e180001, quorum=127.0.0.1:62869, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-22T18:58:31,969 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44109-0x10141087e180000, quorum=127.0.0.1:62869, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T18:58:31,969 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43593-0x10141087e180001, quorum=127.0.0.1:62869, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T18:58:31,969 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T18:58:31,970 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43593-0x10141087e180001, quorum=127.0.0.1:62869, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T18:58:31,970 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:44109-0x10141087e180000, quorum=127.0.0.1:62869, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T18:58:31,970 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-22T18:58:31,970 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T18:58:31,970 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:58:31,970 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'd79ba0c344fb,43593,1732301724173' ***** 2024-11-22T18:58:31,970 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-22T18:58:31,971 INFO [RS:0;d79ba0c344fb:43593 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-22T18:58:31,971 INFO [RS:0;d79ba0c344fb:43593 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-22T18:58:31,971 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-22T18:58:31,971 INFO [RS:0;d79ba0c344fb:43593 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-22T18:58:31,971 INFO [RS:0;d79ba0c344fb:43593 {}] regionserver.HRegionServer(959): stopping server d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:31,971 INFO [RS:0;d79ba0c344fb:43593 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-22T18:58:31,971 INFO [RS:0;d79ba0c344fb:43593 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;d79ba0c344fb:43593. 2024-11-22T18:58:31,971 DEBUG [RS:0;d79ba0c344fb:43593 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T18:58:31,971 DEBUG [RS:0;d79ba0c344fb:43593 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:58:31,972 INFO [RS:0;d79ba0c344fb:43593 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-22T18:58:31,972 INFO [RS:0;d79ba0c344fb:43593 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-22T18:58:31,972 INFO [RS:0;d79ba0c344fb:43593 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-22T18:58:31,972 INFO [RS:0;d79ba0c344fb:43593 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-22T18:58:31,972 INFO [RS:0;d79ba0c344fb:43593 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-22T18:58:31,972 DEBUG [RS:0;d79ba0c344fb:43593 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-22T18:58:31,973 DEBUG [RS:0;d79ba0c344fb:43593 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-22T18:58:31,973 DEBUG [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-22T18:58:31,973 INFO [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-22T18:58:31,973 DEBUG [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-22T18:58:31,973 DEBUG [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-22T18:58:31,973 DEBUG [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-22T18:58:31,973 INFO [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-22T18:58:31,997 DEBUG [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/hbase/meta/1588230740/.tmp/info/e028a63b470d42259069540b6a88f1c3 is 95, key is TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851./info:/1732301759837/DeleteFamily/seqid=0 2024-11-22T18:58:32,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742396_1572 (size=6514) 2024-11-22T18:58:32,063 INFO [regionserver/d79ba0c344fb:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-22T18:58:32,173 DEBUG [RS:0;d79ba0c344fb:43593 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-22T18:58:32,373 DEBUG [RS:0;d79ba0c344fb:43593 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-22T18:58:32,401 INFO [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/hbase/meta/1588230740/.tmp/info/e028a63b470d42259069540b6a88f1c3 2024-11-22T18:58:32,420 DEBUG [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/hbase/meta/1588230740/.tmp/ns/f0bc90254e424c6fb468619bfb8907b5 is 93, key is TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851./ns:/1732301759797/DeleteFamily/seqid=0 2024-11-22T18:58:32,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742397_1573 (size=5906) 2024-11-22T18:58:32,573 DEBUG [RS:0;d79ba0c344fb:43593 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-22T18:58:32,769 INFO [regionserver/d79ba0c344fb:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-22T18:58:32,770 INFO [regionserver/d79ba0c344fb:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-22T18:58:32,773 DEBUG [RS:0;d79ba0c344fb:43593 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-22T18:58:32,824 INFO [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/hbase/meta/1588230740/.tmp/ns/f0bc90254e424c6fb468619bfb8907b5 2024-11-22T18:58:32,841 DEBUG [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/hbase/meta/1588230740/.tmp/rep_barrier/24a237dd45204a10b2f6703e4ca381ca is 102, key is TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851./rep_barrier:/1732301759797/DeleteFamily/seqid=0 2024-11-22T18:58:32,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742398_1574 (size=6025) 2024-11-22T18:58:32,974 DEBUG [RS:0;d79ba0c344fb:43593 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-22T18:58:33,174 DEBUG [RS:0;d79ba0c344fb:43593 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-22T18:58:33,245 INFO [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/hbase/meta/1588230740/.tmp/rep_barrier/24a237dd45204a10b2f6703e4ca381ca 2024-11-22T18:58:33,262 DEBUG [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/hbase/meta/1588230740/.tmp/table/c28efb5c48074556ac6a767da34131c3 is 96, key is TestAcidGuarantees,,1732301728228.0fc957929e5be1c291085a12bd8cf851./table:/1732301759797/DeleteFamily/seqid=0 2024-11-22T18:58:33,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742399_1575 (size=5936) 2024-11-22T18:58:33,374 DEBUG [RS:0;d79ba0c344fb:43593 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-22T18:58:33,574 DEBUG [RS:0;d79ba0c344fb:43593 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-22T18:58:33,666 INFO [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/hbase/meta/1588230740/.tmp/table/c28efb5c48074556ac6a767da34131c3 2024-11-22T18:58:33,670 DEBUG [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/hbase/meta/1588230740/.tmp/info/e028a63b470d42259069540b6a88f1c3 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/hbase/meta/1588230740/info/e028a63b470d42259069540b6a88f1c3 2024-11-22T18:58:33,673 INFO [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/hbase/meta/1588230740/info/e028a63b470d42259069540b6a88f1c3, entries=12, sequenceid=90, filesize=6.4 K 2024-11-22T18:58:33,673 DEBUG [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/hbase/meta/1588230740/.tmp/ns/f0bc90254e424c6fb468619bfb8907b5 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/hbase/meta/1588230740/ns/f0bc90254e424c6fb468619bfb8907b5 2024-11-22T18:58:33,676 INFO [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/hbase/meta/1588230740/ns/f0bc90254e424c6fb468619bfb8907b5, entries=8, sequenceid=90, filesize=5.8 K 2024-11-22T18:58:33,676 DEBUG [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/hbase/meta/1588230740/.tmp/rep_barrier/24a237dd45204a10b2f6703e4ca381ca as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/hbase/meta/1588230740/rep_barrier/24a237dd45204a10b2f6703e4ca381ca 2024-11-22T18:58:33,679 INFO [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/hbase/meta/1588230740/rep_barrier/24a237dd45204a10b2f6703e4ca381ca, entries=6, sequenceid=90, filesize=5.9 K 2024-11-22T18:58:33,679 DEBUG [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/hbase/meta/1588230740/.tmp/table/c28efb5c48074556ac6a767da34131c3 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/hbase/meta/1588230740/table/c28efb5c48074556ac6a767da34131c3 2024-11-22T18:58:33,682 INFO [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/hbase/meta/1588230740/table/c28efb5c48074556ac6a767da34131c3, entries=7, sequenceid=90, filesize=5.8 K 2024-11-22T18:58:33,682 INFO [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1709ms, sequenceid=90, compaction requested=false 2024-11-22T18:58:33,688 DEBUG [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-22T18:58:33,688 DEBUG [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-22T18:58:33,688 INFO [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-22T18:58:33,688 DEBUG [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732301911972Running coprocessor pre-close hooks at 1732301911973 (+1 ms)Disabling compacts and flushes for region at 1732301911973Disabling writes for close at 1732301911973Obtaining lock to block concurrent updates at 1732301911973Preparing flush snapshotting stores in 1588230740 at 1732301911973Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1732301911973Flushing stores of hbase:meta,,1.1588230740 at 1732301911974 (+1 ms)Flushing 1588230740/info: creating writer at 1732301911974Flushing 1588230740/info: appending metadata at 1732301911997 (+23 ms)Flushing 1588230740/info: closing flushed file at 1732301911997Flushing 1588230740/ns: creating writer at 1732301912405 (+408 ms)Flushing 1588230740/ns: appending metadata at 1732301912420 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1732301912420Flushing 1588230740/rep_barrier: creating writer at 1732301912827 (+407 ms)Flushing 1588230740/rep_barrier: appending metadata at 1732301912841 (+14 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1732301912841Flushing 1588230740/table: creating writer at 1732301913248 (+407 ms)Flushing 1588230740/table: appending metadata at 1732301913262 (+14 ms)Flushing 1588230740/table: closing flushed file at 1732301913262Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1149d213: reopening flushed file at 1732301913669 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@16e5500: reopening flushed file at 1732301913673 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@436e7cdc: reopening flushed file at 1732301913676 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@200acf89: reopening flushed file at 1732301913679 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1709ms, sequenceid=90, compaction requested=false at 1732301913682 (+3 ms)Writing region close event to WAL at 1732301913685 (+3 ms)Running coprocessor post-close hooks at 1732301913688 (+3 ms)Closed at 1732301913688 2024-11-22T18:58:33,689 DEBUG [RS_CLOSE_META-regionserver/d79ba0c344fb:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-22T18:58:33,775 INFO [RS:0;d79ba0c344fb:43593 {}] regionserver.HRegionServer(976): stopping server d79ba0c344fb,43593,1732301724173; all regions closed. 2024-11-22T18:58:33,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741834_1010 (size=25344) 2024-11-22T18:58:33,781 DEBUG [RS:0;d79ba0c344fb:43593 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/oldWALs 2024-11-22T18:58:33,781 INFO [RS:0;d79ba0c344fb:43593 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL d79ba0c344fb%2C43593%2C1732301724173.meta:.meta(num 1732301727771) 2024-11-22T18:58:33,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741832_1008 (size=13411323) 2024-11-22T18:58:33,784 DEBUG [RS:0;d79ba0c344fb:43593 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/oldWALs 2024-11-22T18:58:33,784 INFO [RS:0;d79ba0c344fb:43593 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL d79ba0c344fb%2C43593%2C1732301724173:(num 1732301726853) 2024-11-22T18:58:33,784 DEBUG [RS:0;d79ba0c344fb:43593 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T18:58:33,784 INFO [RS:0;d79ba0c344fb:43593 {}] regionserver.LeaseManager(133): Closed leases 2024-11-22T18:58:33,784 INFO [RS:0;d79ba0c344fb:43593 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-22T18:58:33,784 INFO [RS:0;d79ba0c344fb:43593 {}] hbase.ChoreService(370): Chore service for: regionserver/d79ba0c344fb:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-22T18:58:33,785 INFO [RS:0;d79ba0c344fb:43593 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-22T18:58:33,785 INFO [regionserver/d79ba0c344fb:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-22T18:58:33,785 INFO [RS:0;d79ba0c344fb:43593 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43593 2024-11-22T18:58:33,789 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43593-0x10141087e180001, quorum=127.0.0.1:62869, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/d79ba0c344fb,43593,1732301724173 2024-11-22T18:58:33,789 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44109-0x10141087e180000, quorum=127.0.0.1:62869, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-22T18:58:33,789 INFO [RS:0;d79ba0c344fb:43593 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-22T18:58:33,791 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [d79ba0c344fb,43593,1732301724173] 2024-11-22T18:58:33,794 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/d79ba0c344fb,43593,1732301724173 already deleted, retry=false 2024-11-22T18:58:33,794 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; d79ba0c344fb,43593,1732301724173 expired; onlineServers=0 2024-11-22T18:58:33,794 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'd79ba0c344fb,44109,1732301723398' ***** 2024-11-22T18:58:33,794 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-22T18:58:33,794 INFO [M:0;d79ba0c344fb:44109 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-22T18:58:33,794 INFO [M:0;d79ba0c344fb:44109 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-22T18:58:33,794 DEBUG [M:0;d79ba0c344fb:44109 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-22T18:58:33,794 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-22T18:58:33,794 DEBUG [M:0;d79ba0c344fb:44109 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-22T18:58:33,795 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster-HFileCleaner.small.0-1732301726500 {}] cleaner.HFileCleaner(306): Exit Thread[master/d79ba0c344fb:0:becomeActiveMaster-HFileCleaner.small.0-1732301726500,5,FailOnTimeoutGroup] 2024-11-22T18:58:33,795 DEBUG [master/d79ba0c344fb:0:becomeActiveMaster-HFileCleaner.large.0-1732301726498 {}] cleaner.HFileCleaner(306): Exit Thread[master/d79ba0c344fb:0:becomeActiveMaster-HFileCleaner.large.0-1732301726498,5,FailOnTimeoutGroup] 2024-11-22T18:58:33,795 INFO [M:0;d79ba0c344fb:44109 {}] hbase.ChoreService(370): Chore service for: master/d79ba0c344fb:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-22T18:58:33,795 INFO [M:0;d79ba0c344fb:44109 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-22T18:58:33,795 DEBUG [M:0;d79ba0c344fb:44109 {}] master.HMaster(1795): Stopping service threads 2024-11-22T18:58:33,795 INFO [M:0;d79ba0c344fb:44109 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-22T18:58:33,795 INFO [M:0;d79ba0c344fb:44109 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T18:58:33,795 ERROR [M:0;d79ba0c344fb:44109 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-22T18:58:33,796 INFO [M:0;d79ba0c344fb:44109 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T18:58:33,796 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-22T18:58:33,796 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44109-0x10141087e180000, quorum=127.0.0.1:62869, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-22T18:58:33,796 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44109-0x10141087e180000, quorum=127.0.0.1:62869, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T18:58:33,797 DEBUG [M:0;d79ba0c344fb:44109 {}] zookeeper.ZKUtil(347): master:44109-0x10141087e180000, quorum=127.0.0.1:62869, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-22T18:58:33,797 WARN [M:0;d79ba0c344fb:44109 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-22T18:58:33,797 INFO [M:0;d79ba0c344fb:44109 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/.lastflushedseqids 2024-11-22T18:58:33,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742400_1576 (size=72) 2024-11-22T18:58:33,854 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-22T18:58:33,854 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-22T18:58:33,855 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-22T18:58:33,892 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43593-0x10141087e180001, quorum=127.0.0.1:62869, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T18:58:33,892 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43593-0x10141087e180001, quorum=127.0.0.1:62869, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T18:58:33,892 INFO [RS:0;d79ba0c344fb:43593 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-22T18:58:33,892 INFO [RS:0;d79ba0c344fb:43593 {}] regionserver.HRegionServer(1031): Exiting; stopping=d79ba0c344fb,43593,1732301724173; zookeeper connection closed. 2024-11-22T18:58:33,893 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@134c88fa {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@134c88fa 2024-11-22T18:58:33,893 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-22T18:58:34,207 INFO [M:0;d79ba0c344fb:44109 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T18:58:34,207 INFO [M:0;d79ba0c344fb:44109 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-22T18:58:34,207 DEBUG [M:0;d79ba0c344fb:44109 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-22T18:58:34,207 INFO [M:0;d79ba0c344fb:44109 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T18:58:34,207 DEBUG [M:0;d79ba0c344fb:44109 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T18:58:34,207 DEBUG [M:0;d79ba0c344fb:44109 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-22T18:58:34,207 DEBUG [M:0;d79ba0c344fb:44109 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T18:58:34,207 INFO [M:0;d79ba0c344fb:44109 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=749.55 KB heapSize=920.38 KB 2024-11-22T18:58:34,223 DEBUG [M:0;d79ba0c344fb:44109 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/10fc8b7236c94197bbd01c97a95c353c is 82, key is hbase:meta,,1/info:regioninfo/1732301727874/Put/seqid=0 2024-11-22T18:58:34,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742401_1577 (size=5672) 2024-11-22T18:58:34,627 INFO [M:0;d79ba0c344fb:44109 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2095 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/10fc8b7236c94197bbd01c97a95c353c 2024-11-22T18:58:34,649 DEBUG [M:0;d79ba0c344fb:44109 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1b32e23d16804546a702824ca0f6197e is 2275, key is \x00\x00\x00\x00\x00\x00\x00'/proc:d/1732301763008/Put/seqid=0 2024-11-22T18:58:34,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742402_1578 (size=46559) 2024-11-22T18:58:34,661 INFO [M:0;d79ba0c344fb:44109 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=748.99 KB at sequenceid=2095 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1b32e23d16804546a702824ca0f6197e 2024-11-22T18:58:34,665 INFO [M:0;d79ba0c344fb:44109 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 1b32e23d16804546a702824ca0f6197e 2024-11-22T18:58:34,680 DEBUG [M:0;d79ba0c344fb:44109 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a499d6aea6d34779a4b06811cc957ad3 is 69, key is d79ba0c344fb,43593,1732301724173/rs:state/1732301726618/Put/seqid=0 2024-11-22T18:58:34,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073742403_1579 (size=5156) 2024-11-22T18:58:35,084 INFO [M:0;d79ba0c344fb:44109 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2095 (bloomFilter=true), to=hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a499d6aea6d34779a4b06811cc957ad3 2024-11-22T18:58:35,087 DEBUG [M:0;d79ba0c344fb:44109 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/10fc8b7236c94197bbd01c97a95c353c as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/10fc8b7236c94197bbd01c97a95c353c 2024-11-22T18:58:35,090 INFO [M:0;d79ba0c344fb:44109 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/10fc8b7236c94197bbd01c97a95c353c, entries=8, sequenceid=2095, filesize=5.5 K 2024-11-22T18:58:35,090 DEBUG [M:0;d79ba0c344fb:44109 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1b32e23d16804546a702824ca0f6197e as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/1b32e23d16804546a702824ca0f6197e 2024-11-22T18:58:35,093 INFO [M:0;d79ba0c344fb:44109 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 1b32e23d16804546a702824ca0f6197e 2024-11-22T18:58:35,093 INFO [M:0;d79ba0c344fb:44109 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/1b32e23d16804546a702824ca0f6197e, entries=184, sequenceid=2095, filesize=45.5 K 2024-11-22T18:58:35,093 DEBUG [M:0;d79ba0c344fb:44109 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a499d6aea6d34779a4b06811cc957ad3 as hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/a499d6aea6d34779a4b06811cc957ad3 2024-11-22T18:58:35,096 INFO [M:0;d79ba0c344fb:44109 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41517/user/jenkins/test-data/f1b054e0-2645-5dd6-e3a1-14e31a637875/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/a499d6aea6d34779a4b06811cc957ad3, entries=1, sequenceid=2095, filesize=5.0 K 2024-11-22T18:58:35,096 INFO [M:0;d79ba0c344fb:44109 {}] regionserver.HRegion(3140): Finished flush of dataSize ~749.55 KB/767539, heapSize ~920.08 KB/942160, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 889ms, sequenceid=2095, compaction requested=false 2024-11-22T18:58:35,098 INFO [M:0;d79ba0c344fb:44109 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T18:58:35,098 DEBUG [M:0;d79ba0c344fb:44109 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732301914207Disabling compacts and flushes for region at 1732301914207Disabling writes for close at 1732301914207Obtaining lock to block concurrent updates at 1732301914207Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732301914207Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=767539, getHeapSize=942400, getOffHeapSize=0, getCellsCount=2304 at 1732301914208 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732301914208Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732301914208Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732301914223 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732301914223Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732301914630 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732301914648 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732301914648Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732301914665 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732301914680 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732301914680Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@59e17975: reopening flushed file at 1732301915087 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1739430d: reopening flushed file at 1732301915090 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@67cdb0b4: reopening flushed file at 1732301915093 (+3 ms)Finished flush of dataSize ~749.55 KB/767539, heapSize ~920.08 KB/942160, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 889ms, sequenceid=2095, compaction requested=false at 1732301915096 (+3 ms)Writing region close event to WAL at 1732301915098 (+2 ms)Closed at 1732301915098 2024-11-22T18:58:35,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741830_1006 (size=904969) 2024-11-22T18:58:35,100 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-22T18:58:35,100 INFO [M:0;d79ba0c344fb:44109 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-22T18:58:35,100 INFO [M:0;d79ba0c344fb:44109 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44109 2024-11-22T18:58:35,101 INFO [M:0;d79ba0c344fb:44109 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-22T18:58:35,203 INFO [M:0;d79ba0c344fb:44109 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-22T18:58:35,203 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44109-0x10141087e180000, quorum=127.0.0.1:62869, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T18:58:35,203 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44109-0x10141087e180000, quorum=127.0.0.1:62869, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T18:58:35,207 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-22T18:58:35,210 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-22T18:58:35,210 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-22T18:58:35,210 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-22T18:58:35,210 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/001b4aa4-1a7f-6591-3cef-5770bc0658f1/hadoop.log.dir/,STOPPED} 2024-11-22T18:58:35,213 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-22T18:58:35,213 WARN [BP-1295690749-172.17.0.2-1732301720241 heartbeating to localhost/127.0.0.1:41517 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-22T18:58:35,213 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-22T18:58:35,213 WARN [BP-1295690749-172.17.0.2-1732301720241 heartbeating to localhost/127.0.0.1:41517 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1295690749-172.17.0.2-1732301720241 (Datanode Uuid d7ea0a5a-d1bc-40cd-b1e0-3faca14101ae) service to localhost/127.0.0.1:41517 2024-11-22T18:58:35,215 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/001b4aa4-1a7f-6591-3cef-5770bc0658f1/cluster_d9590100-86d5-54e6-c888-90717c40c23c/data/data1/current/BP-1295690749-172.17.0.2-1732301720241 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-22T18:58:35,216 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/001b4aa4-1a7f-6591-3cef-5770bc0658f1/cluster_d9590100-86d5-54e6-c888-90717c40c23c/data/data2/current/BP-1295690749-172.17.0.2-1732301720241 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-22T18:58:35,216 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-22T18:58:35,224 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-22T18:58:35,225 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-22T18:58:35,225 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-22T18:58:35,225 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-22T18:58:35,225 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/001b4aa4-1a7f-6591-3cef-5770bc0658f1/hadoop.log.dir/,STOPPED} 2024-11-22T18:58:35,244 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-22T18:58:35,356 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down